Frontend select model

src/components/api/recogComponent.tsx

@@ -17,6 +17,7 @@ import { STTRenderer } from '../sttRenderer';
 import { PlaybackStatus, StreamTextStatus } from '../../react-redux&middleware/redux/types/apiTypes';
 import Swal from 'sweetalert2';
 import { useRecognition } from './returnAPI';
+import { selectSelectedModel } from '../../react-redux&middleware/redux/reducers/modelSelectionReducers';
 
 export const RecogComponent: React.FC = (props) => {
 
@@ -82,9 +83,10 @@ export const RecogComponent: React.FC = (props) => {
    const sRecog = useSelector((state: RootState) => {
       return state.SRecognitionReducer as SRecognition;
    })
+   const selectedModelOption = useSelector(selectSelectedModel);
 
    // if else for whisper transcript, apiStatus for 4=whisper and control status for listening
-   const transcript = useRecognition(sRecog, apiStatus, controlStatus, azureStatus, streamTextStatus, scribearServerStatus, playbackStatus);
+   const transcript = useRecognition(sRecog, apiStatus, controlStatus, azureStatus, streamTextStatus, scribearServerStatus, selectedModelOption, playbackStatus);
    // console.log("Recog component received new transcript: ", transcript)
    return STTRenderer(transcript);
 };

src/components/api/returnAPI.tsx

@@ -1,6 +1,6 @@
 // import * as sdk from 'microsoft-cognitiveservices-speech-sdk'
 import installCOIServiceWorker from './coi-serviceworker'
-import { API, PlaybackStatus} from '../../react-redux&middleware/redux/typesImports';
+import { API, PlaybackStatus } from '../../react-redux&middleware/redux/typesImports';
 import {
    ApiStatus,
    AzureStatus,
@@ -9,7 +9,7 @@ import {
    StreamTextStatus,
    ScribearServerStatus
 } from '../../react-redux&middleware/redux/typesImports';
-import  {useEffect, useState } from 'react';
+import { useEffect, useState } from 'react';
 import { batch, useDispatch, useSelector } from 'react-redux';
 
 import { AzureRecognizer } from './azure/azureRecognizer';
@@ -23,6 +23,7 @@ import { WhisperRecognizer } from './whisper/whisperRecognizer';
 import { PlaybackRecognizer } from './playback/playbackRecognizer';
 
 import { ScribearRecognizer } from './scribearServer/scribearRecognizer';
+import type { SelectedOption } from '../../react-redux&middleware/redux/types/modelSelection';
 // import { PlaybackReducer } from '../../react-redux&middleware/redux/reducers/apiReducers';
 // controls what api to send and what to do when error handling.
 
@@ -55,17 +56,17 @@ export const returnRecogAPI = (api : ApiStatus, control : ControlStatus, azure :
 */
 
 
-const createRecognizer = (currentApi: number, control: ControlStatus, azure: AzureStatus, streamTextConfig: StreamTextStatus, scribearServerStatus: ScribearServerStatus,playbackStatus:PlaybackStatus): Recognizer => {
+const createRecognizer = (currentApi: number, control: ControlStatus, azure: AzureStatus, streamTextConfig: StreamTextStatus, scribearServerStatus: ScribearServerStatus, selectedModelOption: SelectedOption, playbackStatus: PlaybackStatus): Recognizer => {
    if (currentApi === API.SCRIBEAR_SERVER) {
-      return new ScribearRecognizer(scribearServerStatus, control.speechLanguage.CountryCode);
+      return new ScribearRecognizer(scribearServerStatus, selectedModelOption, control.speechLanguage.CountryCode);
    } else if (currentApi === API.PLAYBACK) {
       return new PlaybackRecognizer(playbackStatus);
    }
    if (currentApi === API.WEBSPEECH) {
       return new WebSpeechRecognizer(null, control.speechLanguage.CountryCode);
    } else if (currentApi === API.AZURE_TRANSLATION) {
       return new AzureRecognizer(null, control.speechLanguage.CountryCode, azure);
-   } 
+   }
    else if (currentApi === API.AZURE_CONVERSATION) {
       throw new Error("Not implemented");
    }
@@ -92,9 +93,9 @@ const updateTranscript = (dispatch: Dispatch) => (newFinalBlocks: Array<Transcri
    // batch makes these dispatches only cause one re-rendering
    batch(() => {
       for (const block of newFinalBlocks) {
-         dispatch({type: "transcript/new_final_block", payload: block});
+         dispatch({ type: "transcript/new_final_block", payload: block });
       }
-      dispatch({type: 'transcript/update_in_progress_block', payload: newInProgressBlock});
+      dispatch({ type: 'transcript/update_in_progress_block', payload: newInProgressBlock });
    })
 }
 
@@ -111,8 +112,8 @@ const updateTranscript = (dispatch: Dispatch) => (newFinalBlocks: Array<Transcri
  * 
  * @return transcripts, resetTranscript, recogHandler
  */
-export const useRecognition = (sRecog : SRecognition, api : ApiStatus, control : ControlStatus, 
-   azure : AzureStatus, streamTextConfig : StreamTextStatus, scribearServerStatus, playbackStatus: PlaybackStatus) => {
+export const useRecognition = (sRecog: SRecognition, api: ApiStatus, control: ControlStatus,
+   azure: AzureStatus, streamTextConfig: StreamTextStatus, scribearServerStatus, selectedModelOption: SelectedOption, playbackStatus: PlaybackStatus) => {
 
    const [recognizer, setRecognizer] = useState<Recognizer>();
    // TODO: Add a reset button to utitlize resetTranscript
@@ -129,9 +130,9 @@ export const useRecognition = (sRecog : SRecognition, api : ApiStatus, control :
       console.log("UseRecognition, switching to new recognizer: ", api.currentApi);
 
       let newRecognizer: Recognizer | null;
-      try{
+      try {
          // Create new recognizer, and subscribe to its events
-         newRecognizer = createRecognizer(api.currentApi, control, azure, streamTextConfig, scribearServerStatus, playbackStatus);
+         newRecognizer = createRecognizer(api.currentApi, control, azure, streamTextConfig, scribearServerStatus, selectedModelOption, playbackStatus);
          newRecognizer.onTranscribed(updateTranscript(dispatch));
          setRecognizer(newRecognizer)
 
@@ -148,7 +149,7 @@ export const useRecognition = (sRecog : SRecognition, api : ApiStatus, control :
          // Stop current recognizer when switching to another one, if possible
          newRecognizer?.stop();
       }
-   }, [api.currentApi, azure, control, streamTextConfig, playbackStatus, scribearServerStatus]);
+   }, [api.currentApi, azure, control, streamTextConfig, playbackStatus, scribearServerStatus, selectedModelOption]);
 
    // Start / stop recognizer, if listening toggled
    useEffect(() => {
@@ -173,7 +174,7 @@ export const useRecognition = (sRecog : SRecognition, api : ApiStatus, control :
    }, [azure.phrases]);
 
    // TODO: whisper's transcript is not in redux store but only in sessionStorage at the moment.
-   let transcript : string = useSelector((state: RootState) => {
+   let transcript: string = useSelector((state: RootState) => {
       return state.TranscriptReducer.transcripts[0].toString()
    });
    // if (api.currentApi === API.WHISPER) { 

src/components/api/scribearServer/scribearRecognizer.tsx

@@ -1,7 +1,10 @@
 import { Recognizer } from '../recognizer';
 import { TranscriptBlock } from '../../../react-redux&middleware/redux/types/TranscriptTypes';
 import { ScribearServerStatus } from '../../../react-redux&middleware/redux/typesImports';
-import RecordRTC, {StereoAudioRecorder} from 'recordrtc';
+import RecordRTC, { StereoAudioRecorder } from 'recordrtc';
+import { store } from '../../../store'
+import { setModelOptions, setSelectedModel } from '../../../react-redux&middleware/redux/reducers/modelSelectionReducers';
+import type { SelectedOption } from '../../../react-redux&middleware/redux/types/modelSelection';
 
 
 enum BackendTranscriptBlockType {
@@ -19,29 +22,35 @@ type BackendTranscriptBlock = {
 
 
 export class ScribearRecognizer implements Recognizer {
-    private scribearServerStatus : ScribearServerStatus
-    private socket : WebSocket | null = null
-    private transcribedCallback : any
+    private scribearServerStatus: ScribearServerStatus
+    private selectedModelOption: SelectedOption
+    private socket: WebSocket | null = null
+    private ready = false;
+    private transcribedCallback: any
     private errorCallback?: (e: Error) => void;
-    private language : string
+    private language: string
     private recorder?: RecordRTC;
     private kSampleRate = 16000;
 
+    urlParams = new URLSearchParams(window.location.search);
+    mode = this.urlParams.get('mode');
 
     /**
      * Creates an Azure recognizer instance that listens to the default microphone
      * and expects speech in the given language 
      * @param audioSource Not implemented yet
      * @param language Expected language of the speech to be transcribed
      */
-    constructor(scribearServerStatus: ScribearServerStatus, language:string) {
+    constructor(scribearServerStatus: ScribearServerStatus, selectedModelOption: SelectedOption, language: string) {
         console.log("ScribearRecognizer, new recognizer being created!")
+
         this.language = language;
+        this.selectedModelOption = selectedModelOption;
         this.scribearServerStatus = scribearServerStatus;
     }
 
     private async _startRecording() {
-        let mic_stream = await navigator.mediaDevices.getUserMedia({audio: true, video: false});
+        let mic_stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false });
 
         this.recorder = new RecordRTC(mic_stream, {
             type: 'audio',
@@ -53,7 +62,7 @@ export class ScribearRecognizer implements Recognizer {
             },
             recorderType: StereoAudioRecorder,
             numberOfAudioChannels: 1,
-          });
+        });
 
         this.recorder.startRecording();
     }
@@ -64,26 +73,47 @@ export class ScribearRecognizer implements Recognizer {
      */
     start() {
         console.log("ScribearRecognizer.start()");
-        if(this.socket) {return;}
+        if (this.socket) { return; }
 
         const scribearURL = new URL(this.scribearServerStatus.scribearServerAddress)
-        if (scribearURL.pathname != '/sink') {
+        if (scribearURL.pathname !== '/sink') {
             this._startRecording();
         }
 
         this.socket = new WebSocket(this.scribearServerStatus.scribearServerAddress);
 
-        // this.socket.onopen = (e)=> {...}
+        this.socket.onopen = (event) => {
+            this.socket?.send(JSON.stringify({
+                api_key: this.scribearServerStatus.scribearServerKey,
+                sourceToken: this.scribearServerStatus.scribearServerKey,
+                sessionToken: this.scribearServerStatus.scribearServerSessionToken,
+            }));
+        }
+
         const inProgressBlock = new TranscriptBlock();
-
-        this.socket.onmessage = (event)=> {
+
+        this.socket.onmessage = (event) => {
+            if (!this.ready && this.mode !== 'student') {
+                const message = JSON.parse(event.data);
+                console.log(message);
+                if (message['error'] || !Array.isArray(message)) return;
+
+                store.dispatch(setModelOptions(message));
+
+                if (this.selectedModelOption) {
+                    this.socket?.send(JSON.stringify(this.selectedModelOption));
+                    this.ready = true;
+                }
+                return;
+            }
+
             const server_block: BackendTranscriptBlock = JSON.parse(event.data);
 
             // Todo: extract type of message (inprogress v final) and the text from the message
             const inProgress = server_block.type === BackendTranscriptBlockType.InProgress;
             const text = server_block.text;
 
-            if(inProgress) {
+            if (inProgress) {
                 inProgressBlock.text = text; // replace text
                 this.transcribedCallback([], inProgressBlock);
             } else {
@@ -99,7 +129,7 @@ export class ScribearRecognizer implements Recognizer {
             console.error("WebSocket error event:", event);
             this.errorCallback?.(error);
         };
-        
+
         this.socket.onclose = (event) => {
             console.warn(`WebSocket closed: code=${event.code}, reason=${event.reason}`);
             this.socket = null;
@@ -117,7 +147,7 @@ export class ScribearRecognizer implements Recognizer {
     stop() {
         console.log("ScribearRecognizer.stop()");
         this.recorder?.stopRecording();
-        if(! this.socket) {return;}
+        if (!this.socket) { return; }
         this.socket.close();
         this.socket = null;
     }
@@ -130,7 +160,7 @@ export class ScribearRecognizer implements Recognizer {
     onTranscribed(callback: (newFinalBlocks: Array<TranscriptBlock>, newInProgressBlock: TranscriptBlock) => void) {
         console.log("ScribearRecognizer.onTranscribed()");
         // "recognizing" event signals that the in-progress block has been updated
-       this.transcribedCallback = callback;
+        this.transcribedCallback = callback;
     }
 
     /**

src/components/navbars/sidebar/model/menu.tsx

@@ -0,0 +1,56 @@
+import React from 'react';
+
+import { List, ListItemText, Collapse, ListItem, MemoryIcon, Autocomplete, TextField, Tooltip } from '../../../../muiImports';
+import { useSelector } from 'react-redux';
+import type { RootState } from '../../../../store';
+import { API, type ApiStatus } from '../../../../react-redux&middleware/redux/typesImports';
+import { useDispatch } from 'react-redux';
+import { selectModelOptions, selectSelectedModel, setSelectedModel } from '../../../../react-redux&middleware/redux/reducers/modelSelectionReducers';
+
+export default function ModelMenu(props) {
+  const dispatch = useDispatch();
+  const APIStatus = useSelector((state: RootState) => {
+    return state.APIStatusReducer as ApiStatus;
+  });
+  const modelOptions = useSelector(selectModelOptions);
+  const selected = useSelector(selectSelectedModel);
+  const modelSelectEnable = APIStatus.currentApi !== API.SCRIBEAR_SERVER;
+
+  return (
+    <div>
+      {props.listItemHeader("Model", "model", MemoryIcon)}
+
+      <Collapse in={props.open} timeout="auto" unmountOnExit>
+        <List component="div" disablePadding>
+          <ListItem sx={{ pl: 4 }}>
+            <ListItemText primary="Select Model" />
+          </ListItem>
+
+          <ListItem sx={{ pl: 4 }}>
+            <Autocomplete
+              disabled={modelSelectEnable}
+              sx={{ width: 300 }}
+              disablePortal
+              options={modelOptions}
+              onChange={(_, val) => {
+                dispatch(setSelectedModel(val))
+              }}
+              defaultValue={selected}
+              getOptionLabel={(v) => v.display_name}
+              isOptionEqualToValue={(a, b) => a.model_key === b.model_key}
+              renderInput={(params) => <TextField {...params} />}
+              renderOption={(props, option) => {
+                return <Tooltip key={props.key} title={option.description} placement='right'>
+                  <ListItem {...props}>
+                    {option.display_name}
+                  </ListItem>
+                </Tooltip>
+              }}
+            />
+          </ListItem>
+
+        </List>
+      </Collapse>
+    </div>
+  );
+}

src/components/navbars/sidebar/sidebar.tsx

@@ -6,10 +6,12 @@ import LangMenu from './language/menu';
 import PhraseMenu from './phrase/menu';
 import VisualizationMenu from './audioVisBar/menu';
 import CaptionsMenu from './captions/menu';
+import ModelMenu from './model/menu';
 
 
 export default function SideBar(props) {
   const [state, setState] = React.useState({
+    model: false,
     display: false,
     lang: true,
     visualization: false,
@@ -48,7 +50,10 @@ export default function SideBar(props) {
       <List sx={{ width: { xs: '100vw', sm: '50vw', md: '40vw', lg: '30vw' }, bgcolor: 'background.paper' }} component="nav" aria-labelledby="nested-list-subheader" >
         <LangMenu open={state.lang} isRecording={props.isRecording} listItemHeader={listItemHeader} />
         <Divider/>
-
+
+        <ModelMenu open={state.model} listItemHeader={listItemHeader} />
+        <Divider/>
+
         <DisplayMenu open={state.display} listItemHeader={listItemHeader} />
         <Divider/>
 

src/components/navbars/topbar/api/ScribearServerSettings.tsx

@@ -88,6 +88,29 @@ export default function ScribearServerSettings(props) {
                         }}
                         style={{ width: '100%' }}
                      />
+                     </Box>
+               </ListItem>
+               <ListItem sx={{ pl: 4 }}>
+                  <Box
+                     component="form"
+                     sx={{ width: '50vw',
+                           '& > :not(style)': { pr: '1vw', width: '15vw' },
+                     }}
+                     noValidate
+                     autoComplete="off"
+                     onSubmit={closePopup} // Prevent default submission behavior of refreshing page
+                  >
+                     <TextField
+                        onChange={updateReact}
+                        value={scribearServerStatusBuf.scribearServerKey}
+                        id="scribearServerKey"
+                        label="ScribeAR Server API Key / Token"
+                        variant="outlined"
+                        inputProps={{
+                           placeholder: 'Enter ScribeAR API Key or Token',
+                        }}
+                        style={{ width: '100%' }}
+                     />
                   </Box>
                </ListItem>