diff --git a/analysis/online_analysis.py b/analysis/online_analysis.py index 99db0e20..37f38dcb 100644 --- a/analysis/online_analysis.py +++ b/analysis/online_analysis.py @@ -296,9 +296,8 @@ def cleanup(self): class SaccadeAnalysisWorker(BehaviorAnalysisWorker): ''' - Plots eye, cursor, and target data from experiments that have them. Performs automatic - calibration of eye data to target locations when the cursor enters the target if no - calibration coefficients are available. + Plots calibrated_eye, cursor, and target data from experiments that have them. + This is for eye-related task that requires calibrated eye position ''' def __init__(self, task_params, data_queue, calibration_dir='/var/tmp', buffer_time=1, ylim=1, px_per_cm=51.67, **kwargs): @@ -324,6 +323,7 @@ def get_current_pos(self): targets = [(self.target_pos[k], radius, color if v == 1 else 'green') for k, v in self.targets.items() if v] except: targets = [] + return self.cursor_pos, self.calibrated_eye_pos, targets def draw(self): @@ -333,7 +333,7 @@ def draw(self): buffer = self.task_params['fixation_radius_buffer'] elif 'fixation_dist' in self.task_params: buffer = self.task_params['fixation_dist'] - self.task_params['target_radius'] - eye_radius = 0.2 + eye_radius = 0.1 patches1 = [plt.Circle(pos, radius+buffer) for pos, radius, _ in targets] patches2 = [plt.Circle(cursor_pos, cursor_radius), plt.Circle(calibrated_eye_pos, eye_radius)] @@ -352,6 +352,117 @@ def draw(self): self.diam_plot.set_data(np.arange(len(self.eye_diam)) * 1/(int(self.task_params['fps'])) - self.buffer_time, self.eye_diam[:, 2]/self.px_per_cm) +class EyeHandAnalysisWorker(SaccadeAnalysisWorker): + ''' + Plots calibrated_eye, cursor, and target data from experiments that have them. + This is for eye-hand task + ''' + + def init(self): + super().init() + self.hand_targets = {} + self.eye_targets = {} + self.target_pos = [] + self.target_idx_trial = [] + + def handle_data(self, key, values): + #super().handle_data(key, values) + if key == 'sync_event': + event_name, event_data = values + if event_name == 'TARGET_ON': + self.hand_targets[event_data] = 1 # event data represents target index in bmi3d + #self.eye_targets[self.target_idx_trial[0]] = 1 # bacause eye initial target and hand target appear at the same time + elif event_name == 'TARGET_OFF': + self.hand_targets[event_data] = 0 + elif event_name == 'EYE_TARGET_ON': + self.eye_targets[event_data] = 1 + elif event_name == 'EYE_TARGET_OFF': + self.eye_targets[event_data] = 0 + + if self.task_params['experiment_name'] == 'EyeConstrainedReachingTask': + self.hand_targets[self.target_idx_trial[-1]] = 0 # In this task, the hand target also disappear + + elif event_name in ['PAUSE', 'TRIAL_END', 'HOLD_PENALTY', 'DELAY_PENALTY', 'TIMEOUT_PENALTY','FIXATION_PENALTY','OTHER_PENALTY']: + # Clear targets at the end of the trial + self.hand_targets = {} + self.eye_targets = {} + self.target_pos = [] + self.target_idx_trial = [] + + elif event_name == 'REWARD': + # Set all active targets to reward + for target_idx in self.hand_targets.keys(): + self.hand_targets[target_idx] = 2 if self.hand_targets[target_idx] else 0 + for target_idx in self.eye_targets.keys(): + self.eye_targets[target_idx] = 2 if self.eye_targets[target_idx] else 0 + + elif key == 'cursor': + self.cursor_pos = np.array(values[0])[[0,2]] + elif key == 'calibrated_eye_pos': + self.calibrated_eye_pos = np.array(values[0])[:2] + + # Update eye diameter + if self.calibrated_eye_pos.size > 2: + self.temp = np.array(values[0])[[0,1,4]] + self.eye_diam = np.roll(self.eye_diam, -1, axis=0) + self.eye_diam[-1] = self.temp + + elif key == 'target_location': + target_idx, target_location = values + self.target_pos.append(np.array(target_location)[[0,2]]) + self.target_idx_trial.append(target_idx) + + + def get_current_pos(self): + ''' + Get the current cursor, eye, and target positions + + Returns: + cursor_pos ((2,) tuple): Current cursor position + eye_pos ((2,) tuple): Current eye position and diameters + targets (list): List of active targets in (position, radius, color) format + ''' + try: + radius = self.task_params['target_radius'] + eye_radius = self.task_params['fixation_radius'] + color = 'orange' + eye_color = 'lightskyblue' + eye_targets = [(self.target_pos[0], eye_radius, eye_color if v == 1 else 'green') for k, v in self.eye_targets.items() if v and k < 3] + eye_targets.extend([(self.target_pos[1], eye_radius, eye_color if v == 1 else 'green') for k, v in self.eye_targets.items() if v and k >= 3]) + hand_targets = [(self.target_pos[2], radius, color if v == 1 else 'green') for k, v in self.hand_targets.items() if v] + except: + eye_targets = [] + hand_targets = [] + + return self.cursor_pos, self.calibrated_eye_pos, eye_targets, hand_targets + + def draw(self): + cursor_pos, calibrated_eye_pos, eye_targets, hand_targets = self.get_current_pos() + cursor_radius = self.task_params.get('cursor_radius', 0.25) + if 'fixation_radius_buffer' in self.task_params: + buffer = self.task_params['fixation_radius_buffer'] + elif 'fixation_dist' in self.task_params: + buffer = self.task_params['fixation_dist'] - self.task_params['target_radius'] + eye_radius = 0.2 + + patches1 = [plt.Circle(pos, radius+buffer) for pos, radius, _ in eye_targets] + patches2 = [plt.Circle(cursor_pos, cursor_radius), plt.Circle(calibrated_eye_pos, eye_radius)] + patches3 = [plt.Circle(pos, radius) for pos, radius, _ in eye_targets] + patches4 = [plt.Circle(pos, radius) for pos, radius, _ in hand_targets] + patches = patches1 + patches2 + patches3 + patches4 + self.circles.set_paths(patches) + colors = [[0.8,0.8,0.8] for _, _, c in eye_targets] + ['darkblue', 'darkgreen'] + [c for _, _, c in eye_targets] + [c for _, _, c in hand_targets] + self.circles.set_facecolor(colors) + self.circles.set_alpha(0.5) + + # Update eye diameter plot + self.x_plot.set_data(np.arange(len(self.eye_diam)) * 1/(int(self.task_params['fps'])) - self.buffer_time, + self.eye_diam[:, 0]) + self.y_plot.set_data(np.arange(len(self.eye_diam)) * 1/(int(self.task_params['fps'])) - self.buffer_time, + self.eye_diam[:, 1]) + self.diam_plot.set_data(np.arange(len(self.eye_diam)) * 1/(int(self.task_params['fps'])) - self.buffer_time, + self.eye_diam[:, 2]/self.px_per_cm) + class ERPAnalysisWorker(AnalysisWorker): ''' Plots ERP data from experiments with an ECoG244 array. Automatically calculates @@ -646,6 +757,12 @@ def init(self): elif self.task_params['experiment_name'] == 'SaccadeTask': self.analysis_workers.append((SaccadeAnalysisWorker(self.task_params, data_queue), data_queue)) + elif self.task_params['experiment_name'] == 'HandConstrainedSaccadeTask': + self.analysis_workers.append((EyeHandAnalysisWorker(self.task_params, data_queue), data_queue)) + + elif self.task_params['experiment_name'] == 'EyeConstrainedReachingTask': + self.analysis_workers.append((EyeHandAnalysisWorker(self.task_params, data_queue), data_queue)) + # Is there ecube neural data? if 'record_headstage' in self.task_params and self.task_params['record_headstage']: data_queue = mp.Queue() diff --git a/built_in_tasks/manualcontrolmultitasks.py b/built_in_tasks/manualcontrolmultitasks.py index 92ed268e..7047619b 100644 --- a/built_in_tasks/manualcontrolmultitasks.py +++ b/built_in_tasks/manualcontrolmultitasks.py @@ -10,7 +10,7 @@ from .target_graphics import * from .target_capture_task import ScreenTargetCapture from .target_capture_task_xt import ScreenReachAngle, ScreenReachLine, SequenceCapture, ScreenTargetCapture_ReadySet -from .target_capture_task_eye import EyeConstrainedTargetCapture, HandConstrainedEyeCapture, ScreenTargetCapture_Saccade +from .target_capture_task_eye import EyeConstrainedTargetCapture, HandConstrainedEyeCapture, EyeConstrainedHandCapture, EyeHandSequenceCapture, ScreenTargetCapture_Saccade from .target_tracking_task import ScreenTargetTracking from .rotation_matrices import * @@ -214,6 +214,18 @@ class HandConstrainedSaccadeTask(ManualControlMixin, HandConstrainedEyeCapture): ''' pass +class EyeHandSequenceTask(ManualControlMixin, EyeHandSequenceCapture): + ''' + Saccade task while holding different targets by hand + ''' + pass + +class EyeConstrainedReachingTask(ManualControlMixin, EyeConstrainedHandCapture): + ''' + Saccade and reaching task while holding different targets by eye and hand + ''' + pass + class SaccadeTask(ManualControlMixin, ScreenTargetCapture_Saccade): ''' Center out saccade task. The controller for the cursor is eye positions. The target color changes when subjects fixate the target. diff --git a/built_in_tasks/target_capture_task.py b/built_in_tasks/target_capture_task.py index 9e5d59c7..a953deb5 100644 --- a/built_in_tasks/target_capture_task.py +++ b/built_in_tasks/target_capture_task.py @@ -297,8 +297,8 @@ class ScreenTargetCapture(TargetCapture, Window): limit2d = traits.Bool(True, desc="Limit cursor movement to 2D") sequence_generators = [ - 'out_2D', 'out_2D_select','centerout_2D', 'centeroutback_2D', 'centerout_2D_select', 'rand_target_chain_2D', 'rand_same_target_chain_2D', - 'rand_target_chain_3D', 'corners_2D', 'centerout_tabletop', + 'out_2D', 'out_2D_select', 'centerout_2D', 'centeroutback_2D', 'centerout_2D_select', 'rand_target_chain_2D', 'rand_same_target_chain_2D', + 'rand_target_chain_3D', 'corners_2D', 'centerout_tabletop', 'out_2D_square', 'centerout_2D_square' ] hidden_traits = ['cursor_color', 'target_color', 'cursor_bounds', 'cursor_radius', 'plant_hide_rate', 'starting_pos'] @@ -586,6 +586,39 @@ def out_2D(nblocks=100, ntargets=8, distance=10, origin=(0,0,0)): ]).T yield [idx], [pos + origin] + @staticmethod + def out_2D_square(nblocks=100, width=10, height=10, origin=(0,0,0)): + ''' + Generates a sequence of 2D (x and z) targets at a point on the side of the square + ''' + ntargets = 8 + rng = np.random.default_rng() + for _ in range(nblocks): + order = np.arange(ntargets) + 1 # target indices, starting from 1 + rng.shuffle(order) + + for t in range(ntargets): + idx = order[t] + + if idx == 1: + pos = np.array([0,0,height/2]).T + elif idx == 2: + pos = np.array([width/2,0,height/2]).T + elif idx == 3: + pos = np.array([width/2,0,0]).T + elif idx == 4: + pos = np.array([width/2,0,-height/2]).T + elif idx == 5: + pos = np.array([0,0,-height/2]).T + elif idx == 6: + pos = np.array([-width/2,0,-height/2]).T + elif idx == 7: + pos = np.array([-width/2,0,0]).T + elif idx == 8: + pos = np.array([-width/2,0,height/2]).T + + yield [idx], [pos + origin] + @staticmethod def centerout_2D(nblocks=100, ntargets=8, distance=10, origin=(0,0,0)): ''' @@ -638,6 +671,25 @@ def out_2D_select(nblocks=100, ntargets=8, distance=10, origin=(0,0,0), target_i except StopIteration: break + @staticmethod + def centerout_2D_square(nblocks=100, width=10, height=10, origin=(0,0,0)): + ''' + Pairs of central targets at the origin and peripheral targets centered around the origin + + Returns + ------- + [nblocks*ntargets x 1] array of tuples containing trial indices and [2 x 3] target coordinates + ''' + ntargets = 8 + gen = ScreenTargetCapture.out_2D_square(nblocks, width, height, origin) + for _ in range(nblocks*ntargets): + idx, pos = next(gen) + targs = np.zeros([2, 3]) + origin + targs[1,:] = pos[0] + indices = np.zeros([2,1]) + indices[1] = idx + yield indices, targs + @staticmethod def centeroutback_2D(nblocks=100, ntargets=8, distance=10, origin=(0,0,0)): ''' diff --git a/built_in_tasks/target_capture_task_eye.py b/built_in_tasks/target_capture_task_eye.py index edce1995..580ffef1 100644 --- a/built_in_tasks/target_capture_task_eye.py +++ b/built_in_tasks/target_capture_task_eye.py @@ -11,37 +11,56 @@ class EyeConstrainedTargetCapture(ScreenTargetCapture): ''' - Add a penalty state when subjects looks away. + Fixation requirement is added before go cue ''' fixation_penalty_time = traits.Float(0., desc="Time in fixation penalty state") - fixation_target_color = traits.OptionsList("cyan", *target_colors, desc="Color of the center target under fixation state", bmi3d_input_options=list(target_colors.keys())) + fixation_target_color = traits.OptionsList("fixation_color", *target_colors, desc="Color of the center target under fixation state", bmi3d_input_options=list(target_colors.keys())) + eye_target_color = traits.OptionsList("eye_color", *target_colors, desc="Color of the center target under fixation state", bmi3d_input_options=list(target_colors.keys())) fixation_radius_buffer = traits.Float(.5, desc="additional radius for eye target") + hand_target_color = traits.OptionsList("yellow", *target_colors, desc="Color for the hand-only target", bmi3d_input_options=list(target_colors.keys())) + fixation_radius = traits.Float(.5, desc="additional radius for eye target") status = dict( - wait = dict(start_trial="target"), - target = dict(timeout="timeout_penalty",gaze_target="fixation"), - fixation = dict(enter_target="hold", fixation_break="target"), - hold = dict(leave_target="hold_penalty", hold_complete="delay", fixation_break="fixation_penalty"), - delay = dict(leave_target="delay_penalty", delay_complete="targ_transition", fixation_break="fixation_penalty"), - targ_transition = dict(trial_complete="reward", trial_abort="wait", trial_incomplete="target"), - timeout_penalty = dict(timeout_penalty_end="targ_transition", end_state=True), - hold_penalty = dict(hold_penalty_end="targ_transition", end_state=True), - delay_penalty = dict(delay_penalty_end="targ_transition", end_state=True), - fixation_penalty = dict(fixation_penalty_end="targ_transition",end_state=True), - reward = dict(reward_end="wait", stoppable=False, end_state=True) + wait = dict(start_trial="target", start_pause="pause"), + target = dict(timeout="timeout_penalty", gaze_enter_target="hold", start_pause="pause"), + hold = dict(leave_target="hold_penalty", hold_complete="delay", fixation_break="fixation_penalty", start_pause="pause"), + delay = dict(leave_target="delay_penalty", delay_complete="targ_transition", fixation_break="fixation_penalty", start_pause="pause"), + targ_transition = dict(trial_complete="reward", trial_abort="wait", trial_incomplete="target", start_pause="pause"), + timeout_penalty = dict(timeout_penalty_end="wait", start_pause="pause", end_state=True), + hold_penalty = dict(hold_penalty_end="wait", start_pause="pause", end_state=True), + delay_penalty = dict(delay_penalty_end="wait", start_pause="pause", end_state=True), + fixation_penalty = dict(fixation_penalty_end="wait", start_pause="pause", end_state=True), + reward = dict(reward_end="wait", start_pause="pause", stoppable=False, end_state=True), + pause = dict(end_pause="wait", end_state=True), ) - - def _test_gaze_target(self,ts): - ''' - Check whether eye positions are within the fixation distance - Only apply this to the first target (1st target) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Instantiate the targets + instantiate_targets = kwargs.pop('instantiate_targets', True) + if instantiate_targets: + + target3 = VirtualRectangularTarget(target_width=self.fixation_radius, target_height=self.fixation_radius/2, target_color=target_colors[self.eye_target_color]) + self.targets_eye = [target3] + self.offset_cube = np.array([0,10,self.fixation_radius/2]) # To center the cube target + + def _test_gaze_enter_target(self,ts): ''' - if self.target_index <= 0: - d = np.linalg.norm(self.calibrated_eye_pos) - return d < self.target_radius + self.fixation_radius_buffer + Check whether eye positions and hand cursor are within the target radius + ''' + eye_pos = self.calibrated_eye_pos + eye_d = np.linalg.norm(eye_pos - self.targs[self.target_index,[0,2]]) + + cursor_pos = self.plant.get_endpoint_pos() + hand_d = np.linalg.norm(cursor_pos - self.targs[self.target_index]) + + # Fixation requirement is only for the center target + if self.target_index == 0: + return (eye_d <= self.target_radius + self.fixation_radius_buffer) and (hand_d <= self.target_radius - self.cursor_radius) else: - return True + return hand_d <= self.target_radius - self.cursor_radius def _test_fixation_break(self,ts): ''' @@ -49,72 +68,118 @@ def _test_fixation_break(self,ts): Only apply this to the first hold and delay period ''' if self.target_index <= 0: - d = np.linalg.norm(self.calibrated_eye_pos) - return (d > self.target_radius + self.fixation_radius_buffer) + eye_d = np.linalg.norm(self.calibrated_eye_pos) + return (eye_d > self.target_radius + self.fixation_radius_buffer) def _test_fixation_penalty_end(self,ts): return (ts > self.fixation_penalty_time) - + def _start_wait(self): super()._start_wait() - self.num_fixation_state = 0 # Initializa fixation state + + if self.calc_trial_num() == 0: + + # Instantiate the targets here so they don't show up in any states that might come before "wait" + for target in self.targets_eye: + for model in target.graphics_models: + self.add_model(model) + target.hide() def _start_target(self): - if self.num_fixation_state == 0: - super()._start_target() # target index shouldn't be incremented after fixation break loop - else: - self.sync_event('FIXATION', 0) - self.targets[0].reset() # reset target color after fixation break + super()._start_target() - def _start_fixation(self): - self.num_fixation_state = 1 - self.targets[0].sphere.color = target_colors[self.fixation_target_color] # change target color in fixation state if self.target_index == 0: - self.sync_event('FIXATION', 1) - + self.targets_eye[0].move_to_position(self.targs[self.target_index] - self.offset_cube) + self.targets_eye[0].show() + + def _while_target(self): + if self.target_index == 0: + eye_pos = self.calibrated_eye_pos + eye_d = np.linalg.norm(eye_pos - self.targs[self.target_index,[0,2]]) + if eye_d <= (self.target_radius + self.fixation_radius_buffer): + self.targets_eye[0].cube.color = target_colors[self.fixation_target_color] # chnage color in fixating center + else: + self.targets_eye[0].reset() + + def _start_delay(self): + next_idx = (self.target_index + 1) + if next_idx < self.chain_length: # This is for hand target in the second delay + self.targets[next_idx].move_to_position(self.targs[next_idx]) + self.targets[next_idx].sphere.color = target_colors[self.hand_target_color] + self.targets[next_idx].show() + self.sync_event('TARGET_ON', self.gen_indices[next_idx]) + + def _start_targ_transition(self): + super()._start_targ_transition() + + if self.target_index + 1 < self.chain_length: + # Hide the current target if there are more + self.targets_eye[0].hide() + + def _start_hold_penalty(self): + super()._start_hold_penalty() + self.targets_eye[0].hide() + self.targets_eye[0].reset() + + def _start_delay_penalty(self): + super()._start_delay_penalty() + self.targets_eye[0].hide() + self.targets_eye[0].reset() + def _start_timeout_penalty(self): super()._start_timeout_penalty() - self.num_fixation_state = 0 - - def _start_hold(self): - super()._start_hold() - self.num_fixation_state = 0 # because target state comes again after hold state in a trial + self.targets_eye[0].hide() + self.targets_eye[0].reset() def _start_fixation_penalty(self): + if hasattr(super(), '_start_fixation_penalty'): + super()._start_fixation_penalty() + self._increment_tries() self.sync_event('FIXATION_PENALTY') + self.penalty_index = 1 + self.num_fixation_state = 0 # Hide targets for target in self.targets: target.hide() target.reset() + self.targets_eye[0].hide() + self.targets_eye[0].reset() + def _end_fixation_penalty(self): self.sync_event('TRIAL_END') class HandConstrainedEyeCapture(ScreenTargetCapture): ''' Saccade task with holding another target with hand. Subjects need to hold an initial target with their hand. - Then they need to fixate the first eye target and make a saccade for the second eye target + Then they need to fixate the first eye target and make a saccade for the second eye target. 2 of chain_length is only tested. ''' fixation_radius = traits.Float(2.5, desc="Distance from center that is considered a broken fixation") fixation_penalty_time = traits.Float(1.0, desc="Time in fixation penalty state") - fixation_target_color = traits.OptionsList("cyan", *target_colors, desc="Color of the eye target under fixation state", bmi3d_input_options=list(target_colors.keys())) - eye_target_color = traits.OptionsList("white", *target_colors, desc="Color of the eye target", bmi3d_input_options=list(target_colors.keys())) + fixation_target_color = traits.OptionsList("fixation_color", *target_colors, desc="Color of the eye target under fixation state", bmi3d_input_options=list(target_colors.keys())) + eye_target_color = traits.OptionsList("eye_color", *target_colors, desc="Color of the eye target", bmi3d_input_options=list(target_colors.keys())) fixation_radius_buffer = traits.Float(.5, desc="additional radius for eye target") + fixation_time = traits.Float(.2, desc="additional radius for eye target") + incorrect_target_radius_buffer = traits.Float(.5, desc="additional radius for eye target") + incorrect_target_penalty_time = traits.Float(1, desc="Length of penalty time for acquiring an incorrect target") + exclude_parent_traits = ['hold_time'] status = dict( - wait = dict(start_trial="target", start_pause="pause"), - target = dict(start_pause="pause", leave_target2="hold_penalty",timeout="timeout_penalty",enter_target="hold"), - hold = dict(start_pause="pause", leave_target2="hold_penalty",leave_target="target", gaze_target="fixation"), # must hold an initial hand-target and eye-target - fixation = dict(start_pause="pause", leave_target="delay_penalty",hold_complete="delay", fixation_break="fixation_penalty"), # must hold an initial hand-target and eye-target to initiate a trial + wait = dict(start_trial="init_target", start_pause="pause"), + init_target = dict(enter_target="target", start_pause="pause"), + target = dict(start_pause="pause", timeout="timeout_penalty", return_init_target='init_target', gaze_enter_target="fixation"), + target_eye = dict(start_pause="pause", timeout="timeout_penalty", leave_target='hold_penalty', gaze_target="fixation", gaze_incorrect_target="incorrect_target_penalty"), + fixation = dict(start_pause="pause", leave_target="hold_penalty", fixation_complete="delay", fixation_break="fixation_penalty"), delay = dict(leave_target="delay_penalty", delay_complete="targ_transition", fixation_break="fixation_penalty", start_pause="pause"), - targ_transition = dict(trial_complete="reward", trial_abort="wait", trial_incomplete="target", start_pause="pause"), - timeout_penalty = dict(timeout_penalty_end="targ_transition", start_pause="pause", end_state=True), - hold_penalty = dict(hold_penalty_end="targ_transition", start_pause="pause", end_state=True), - delay_penalty = dict(delay_penalty_end="targ_transition", start_pause="pause", end_state=True), - fixation_penalty = dict(fixation_penalty_end="targ_transition", start_pause="pause", end_state=True), + targ_transition = dict(trial_complete="reward", trial_abort="wait", trial_incomplete="target_eye", start_pause="pause"), + timeout_penalty = dict(timeout_penalty_end="wait", start_pause="pause", end_state=True), + hold_penalty = dict(hold_penalty_end="wait", start_pause="pause", end_state=True), + delay_penalty = dict(delay_penalty_end="wait", start_pause="pause", end_state=True), + fixation_penalty = dict(fixation_penalty_end="wait", start_pause="pause", end_state=True), + incorrect_target_penalty = dict(incorrect_target_penalty_end="wait", start_pause="pause", end_state=True), reward = dict(reward_end="wait", start_pause="pause", stoppable=False, end_state=True), pause = dict(end_pause="wait", end_state=True), ) @@ -129,46 +194,71 @@ def __init__(self, *args, **kwargs): if instantiate_targets: # Target 1 and 2 are for saccade. Target 3 is for hand - target1 = VirtualCircularTarget(target_radius=self.fixation_radius, target_color=target_colors[self.eye_target_color]) - target2 = VirtualCircularTarget(target_radius=self.fixation_radius, target_color=target_colors[self.eye_target_color]) + target1 = VirtualRectangularTarget(target_width=self.fixation_radius, target_height=self.fixation_radius/2, target_color=target_colors[self.eye_target_color]) + target2 = VirtualRectangularTarget(target_width=self.fixation_radius, target_height=self.fixation_radius/2, target_color=target_colors[self.eye_target_color]) target3 = VirtualCircularTarget(target_radius=self.target_radius, target_color=target_colors[self.target_color]) self.targets = [target1, target2] self.targets_hand = [target3] + + self.offset_cube = np.array([0,0,self.fixation_radius/2]) # To center the cube target def _parse_next_trial(self): '''Check that the generator has the required data''' - self.gen_indices, self.targs = self.next_trial # 2 target positions for hand and eye + # 2 target positions for hand and eye. The first and second target index is for eye, and the third one is for hand + self.gen_indices, self.targs = self.next_trial # Update the data sinks with trial information - self.trial_record['trial'] = self.calc_trial_num() # TODO save both eye and hand target positions + self.trial_record['trial'] = self.calc_trial_num() for i in range(len(self.gen_indices)): self.trial_record['index'] = self.gen_indices[i] self.trial_record['target'] = self.targs[i] self.sinks.send("trials", self.trial_record) - def _test_gaze_target(self,ts): - ''' - Check whether eye positions from a target are within the fixation distance + def _test_gaze_enter_target(self,ts): ''' - # Distance of an eye position from a target position + Check whether eye positions and hand cursor are within the target radius + ''' eye_pos = self.calibrated_eye_pos - target_pos = np.delete(self.targs[self.target_index],1) - d_eye = np.linalg.norm(eye_pos - target_pos) - return (d_eye <= self.fixation_radius + self.fixation_radius_buffer) or self.pause + eye_d = np.linalg.norm(eye_pos - self.targs[self.target_index,[0,2]]) + + cursor_pos = self.plant.get_endpoint_pos() + hand_d = np.linalg.norm(cursor_pos - self.targs[-1]) + return (eye_d <= self.fixation_radius + self.fixation_radius_buffer) and (hand_d <= self.target_radius - self.cursor_radius) + + def _test_gaze_target(self, ts): + ''' + Check whether eye position is within the target radius + ''' + eye_pos = self.calibrated_eye_pos + eye_d = np.linalg.norm(eye_pos - self.targs[self.target_index,[0,2]]) + + return eye_d <= self.fixation_radius + self.fixation_radius_buffer + + def _test_gaze_incorrect_target(self, ts): + ''' + Check whether eye position is within the different target (hand target) + ''' + eye_pos = self.calibrated_eye_pos + eye_d = np.linalg.norm(eye_pos - self.targs[-1,[0,2]]) + + return eye_d <= self.target_radius + self.incorrect_target_radius_buffer + def _test_fixation_break(self,ts): ''' Triggers the fixation_penalty state when eye positions are outside fixation distance ''' # Distance of an eye position from a target position eye_pos = self.calibrated_eye_pos - target_pos = np.delete(self.targs[self.target_index],1) - d_eye = np.linalg.norm(eye_pos - target_pos) - return (d_eye > self.fixation_radius + self.fixation_radius_buffer) or self.pause + d_eye = np.linalg.norm(eye_pos - self.targs[self.target_index,[0,2]]) + return d_eye > self.fixation_radius + self.fixation_radius_buffer + + def _test_fixation_complete(self,ts): + return ts > self.fixation_time def _test_fixation_penalty_end(self,ts): - return (ts > self.fixation_penalty_time) + return ts > self.fixation_penalty_time def _test_enter_target(self, ts): ''' @@ -176,7 +266,7 @@ def _test_enter_target(self, ts): ''' cursor_pos = self.plant.get_endpoint_pos() d = np.linalg.norm(cursor_pos - self.targs[-1]) # hand must be within the initial target - return d <= (self.target_radius - self.cursor_radius) or self.pause + return d <= self.target_radius - self.cursor_radius def _test_leave_target(self, ts): ''' @@ -184,24 +274,27 @@ def _test_leave_target(self, ts): ''' cursor_pos = self.plant.get_endpoint_pos() d = np.linalg.norm(cursor_pos - self.targs[-1]) # hand must be within the initial target - return d > (self.target_radius - self.cursor_radius) or self.pause + return d > self.target_radius - self.cursor_radius - def _test_leave_target2(self, ts): + def _test_return_init_target(self, ts): ''' - return true if cursor moves outside the exit radius (This is for the second target state) + return true if cursor moves outside the exit radius, but only applied when the target index is 0. ''' - if self.target_index > 0: - cursor_pos = self.plant.get_endpoint_pos() - d = np.linalg.norm(cursor_pos - self.targs[-1]) # hand must be within the initial target - return d > (self.target_radius - self.cursor_radius) or self.pause + cursor_pos = self.plant.get_endpoint_pos() + d = np.linalg.norm(cursor_pos - self.targs[-1]) + return (d > self.target_radius - self.cursor_radius) and self.target_index == 0 + + def _test_trial_incomplete(self, ts): + return self.target_index < self.chain_length + + def _test_incorrect_target_penalty_end(self, ts): + return ts > self.incorrect_target_penalty_time def _start_wait(self): super()._start_wait() # Redefine chain length because targs in this task has both eye and hand targets self.chain_length = len(self.targets) - - # Initialize fixation state - self.num_hold_state = 0 + self.isfixation_state = False if self.calc_trial_num() == 0: @@ -216,77 +309,65 @@ def _start_wait(self): self.add_model(model) target.hide() - def _start_target(self): - if self.num_hold_state == 0: - self.target_index += 1 # target index shouldn't be incremented after hold break loop - - # Show target if it is hidden (this is the first target, or previous state was a penalty) + def _start_init_target(self): + # Only show the hand target + if self.target_index == -1: target_hand = self.targets_hand[0] - if self.target_index == 0: - target_hand.move_to_position(self.targs[-1]) - target_hand.show() - self.sync_event('TARGET_ON', self.gen_indices[-1]) - - else: - target = self.targets[self.target_index % 2] - target.hide() # hide hand target - self.sync_event('EYE_TARGET_OFF', self.gen_indices[self.target_index % 2]) + target_hand.move_to_position(self.targs[-1]) + target_hand.show() + self.sync_event('TARGET_ON', self.gen_indices[-1]) # the hand target is on - def _start_hold(self): - #self.sync_event('CURSOR_ENTER_TARGET', self.gen_indices[self.target_index]) - self.num_hold_state = 1 + elif self.target_index == 0: # this is from the target state + target = self.targets[self.target_index] + target.hide() + self.sync_event('EYE_TARGET_OFF', self.gen_indices[self.target_index]) - # Show target if it is hidden (this is the first target, or previous state was a penalty) - target = self.targets[self.target_index % 2] + def _start_target(self): + if self.target_index == -1 and not self.isfixation_state: + self.target_index += 1 + + if self.isfixation_state: + self.target_index += 1 + + # Show the eye target if self.target_index == 0: - target.move_to_position(self.targs[self.target_index]) + target = self.targets[self.target_index] + target.move_to_position(self.targs[self.target_index] - self.offset_cube) target.show() self.sync_event('EYE_TARGET_ON', self.gen_indices[self.target_index]) + def _start_target_eye(self): + self.target_index += 1 + def _start_fixation(self): - self.num_hold_state = 0 - self.targets[self.target_index].sphere.color = target_colors[self.fixation_target_color] # change target color in fixation state + self.isfixation_state = True + self.targets[self.target_index].cube.color = target_colors[self.fixation_target_color] # change target color in fixation state self.sync_event('FIXATION', self.gen_indices[self.target_index]) - def _while_fixation(self): - pass - - def _end_fixation(self): - pass - def _start_delay(self): # Make next target visible unless this is the final target in the trial next_idx = (self.target_index + 1) if next_idx < self.chain_length: - target = self.targets[next_idx % 2] - target.move_to_position(self.targs[next_idx % 2]) + target = self.targets[next_idx] + target.move_to_position(self.targs[next_idx] - self.offset_cube) target.show() - self.sync_event('EYE_TARGET_ON', self.gen_indices[next_idx % 2]) - else: - # This delay state should only last 1 cycle, don't sync anything - pass + self.sync_event('EYE_TARGET_ON', self.gen_indices[next_idx]) def _start_targ_transition(self): - if self.target_index == -1: - - # Came from a penalty state - pass - elif self.target_index + 1 < self.chain_length: + if self.target_index + 1 < self.chain_length: # Hide the current target if there are more - self.targets[self.target_index % 2].hide() + self.targets[self.target_index].hide() self.sync_event('EYE_TARGET_OFF', self.gen_indices[self.target_index]) def _start_timeout_penalty(self): super()._start_timeout_penalty() - self.num_hold_state = 0 for target in self.targets_hand: target.hide() target.reset() def _start_hold_penalty(self): super()._start_hold_penalty() - self.num_hold_state = 0 # Hide targets for target in self.targets_hand: target.hide() @@ -302,6 +383,7 @@ def _start_delay_penalty(self): def _start_fixation_penalty(self): self._increment_tries() self.sync_event('FIXATION_PENALTY') + self.penalty_index = 1 # Hide targets for target in self.targets: @@ -315,8 +397,35 @@ def _start_fixation_penalty(self): def _end_fixation_penalty(self): self.sync_event('TRIAL_END') + def _start_incorrect_target_penalty(self): + self._increment_tries() + self.sync_event('OTHER_PENALTY') + self.penalty_index = 1 + + # Hide targets + for target in self.targets: + target.hide() + target.reset() + + for target in self.targets_hand: + target.cue_trial_end_failure() + target.show() + + def _end_incorrect_target_penalty(self): + self.sync_event('TRIAL_END') + + for target in self.targets_hand: + target.hide() + target.reset() + def _start_reward(self): super()._start_reward() + for target in self.targets_hand: + target.cue_trial_end_success() + + def _end_reward(self): + super()._end_reward() + # Hide targets for target in self.targets_hand: target.hide() @@ -407,6 +516,419 @@ def sac_hand_2d(nblocks=20, ntargets=3, dx=10,offset1=(0,0,-2),offset2=(0,0,6.), yield indices, targs +class EyeConstrainedHandCapture(HandConstrainedEyeCapture): + + status = dict( + wait = dict(start_trial="init_target", start_pause="pause"), + init_target = dict(enter_target="target", start_pause="pause"), + target = dict(start_pause="pause", timeout="timeout_penalty", return_init_target='init_target', gaze_enter_target="fixation", gaze_incorrect_target="incorrect_target_penalty"), + fixation = dict(start_pause="pause", leave_target="hold_penalty", fixation_hold_complete="delay", fixation_break="fixation_penalty"), + delay = dict(leave_target="delay_penalty", delay_complete="targ_transition", fixation_break="fixation_penalty", start_pause="pause"), + targ_transition = dict(trial_complete="reward", trial_abort="wait", trial_incomplete="target", start_pause="pause"), + timeout_penalty = dict(timeout_penalty_end="wait", start_pause="pause", end_state=True), + hold_penalty = dict(hold_penalty_end="wait", start_pause="pause", end_state=True), + delay_penalty = dict(delay_penalty_end="wait", start_pause="pause", end_state=True), + incorrect_target_penalty = dict(incorrect_target_penalty_end="wait", start_pause="pause", end_state=True), + fixation_penalty = dict(fixation_penalty_end="wait", start_pause="pause", end_state=True), + reward = dict(reward_end="wait", start_pause="pause", stoppable=False, end_state=True), + pause = dict(end_pause="wait", end_state=True), + ) + + sequence_generators = ['row_target','sac_hand_2d'] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Instantiate the targets + instantiate_targets = kwargs.pop('instantiate_targets', True) + if instantiate_targets: + + # Target 1 and 2 are for saccade. Target 3 is for hand + target1 = VirtualRectangularTarget(target_width=self.fixation_radius, target_height=self.fixation_radius/2, target_color=target_colors[self.eye_target_color]) + target2 = VirtualRectangularTarget(target_width=self.fixation_radius, target_height=self.fixation_radius/2, target_color=target_colors[self.eye_target_color]) + target3 = VirtualCircularTarget(target_radius=self.target_radius, target_color=target_colors[self.target_color]) + target4 = VirtualCircularTarget(target_radius=self.target_radius, target_color=target_colors[self.target_color]) + + self.targets = [target1, target2] + self.targets_hand = [target3, target4] + + self.offset_cube = np.array([0,10,self.fixation_radius/2]) # To center the cube target + + def _test_gaze_enter_target(self,ts): + ''' + Check whether eye positions and hand cursor are within the target radius + ''' + eye_pos = self.calibrated_eye_pos + eye_d = np.linalg.norm(eye_pos - self.targs[self.target_index,[0,2]]) + + cursor_pos = self.plant.get_endpoint_pos() + hand_d = np.linalg.norm(cursor_pos - self.targs[-1-self.target_index]) #targs[-1] is the first hand target, targ[-2] is the second target + + return (eye_d <= self.fixation_radius + self.fixation_radius_buffer) and (hand_d <= self.target_radius - self.cursor_radius) + + def _test_gaze_incorrect_target(self, ts): + ''' + Check whether eye position is within the different target (hand target). This is only applied to the second target + ''' + eye_pos = self.calibrated_eye_pos + eye_d = np.linalg.norm(eye_pos - self.targs[-1,[0,2]]) + + return eye_d <= (self.target_radius + self.incorrect_target_radius_buffer) and self.target_index == 1 + + def _test_leave_target(self, ts): + ''' + return true if cursor moves outside the exit radius + ''' + cursor_pos = self.plant.get_endpoint_pos() + d = np.linalg.norm(cursor_pos - self.targs[-1-self.target_index]) + return d > self.target_radius - self.cursor_radius + + def _test_fixation_hold_complete(self,ts): + return ts > self.fixation_time + + def _while_target(self): + target = self.targets[self.target_index] + target.move_to_position(self.targs[self.target_index] - self.offset_cube) + + eye_pos = self.calibrated_eye_pos + eye_d = np.linalg.norm(eye_pos - self.targs[self.target_index,[0,2]]) + + if eye_d <= self.fixation_radius + self.fixation_radius_buffer: + self.targets[self.target_index].cube.color = target_colors[self.fixation_target_color] # change target color in fixation state + else: + self.targets[self.target_index].cube.color = target_colors[self.eye_target_color] + + def _start_delay(self): + next_idx = (self.target_index + 1) + if next_idx < self.chain_length: + # Show hand target + target = self.targets_hand[next_idx] + target.move_to_position(self.targs[next_idx]) + target.show() # Don't have to sync event because the second target is shared between hand and eye. + + # Show eye target + target = self.targets[next_idx] + target.move_to_position(self.targs[next_idx] - self.offset_cube) + target.show() + self.sync_event('EYE_TARGET_ON', self.gen_indices[next_idx]) + + def _start_targ_transition(self): + super()._start_targ_transition() + if self.target_index + 1 < self.chain_length: + + # Hide the current hand target + self.targets_hand[self.target_index].hide() + +class EyeHandSequenceCapture(EyeConstrainedTargetCapture): + ''' + Subjects have to gaze at and reach to a target, responding to the eye or hand go cue indivisually in sequence trials. + They need to simultaneously move eye and hand to the target in simultaneous trials. + ''' + + exclude_parent_traits = ['delay_time', 'rand_delay'] + rand_delay1 = traits.Tuple((0.4, 0.7), desc="Delay interval for eye") + rand_delay2 = traits.Tuple((0., 0.7), desc="Delay interval for hand") + fixation_time = traits.Float(0.2, desc='Length of fixation required at targets') + sequence_ratio = traits.Float(0.5, desc='Ratio of sequence trials') + + status = dict( + wait = dict(start_trial="target", start_pause="pause"), + target = dict(timeout="timeout_penalty", gaze_enter_target='fixation', start_pause="pause"), + target_eye = dict(timeout="timeout_penalty", gaze_target="delay", leave_target="hold_penalty", start_pause="pause"), + target_hand = dict(timeout="timeout_penalty", enter_target="hold", fixation_break="fixation_penalty", start_pause="pause"), + target_eye_hand = dict(timeout="timeout_penalty", gaze_enter_target='fixation', start_pause="pause"), + fixation = dict(fixation_complete="delay", leave_target="hold_penalty", fixation_break="fixation_penalty", start_pause="pause"), + hold = dict(hold_complete="delay", leave_target="hold_penalty", fixation_break="fixation_penalty", start_pause="pause"), + delay = dict(delay_complete="targ_transition", leave_target="delay_penalty", fixation_break="fixation_penalty", start_pause="pause"), + targ_transition = dict(trial_complete="reward", trial_abort="wait", targ_simultaneous="target_eye_hand",\ + targ_first_sequence="target_eye", targ_second_sequence="target_hand", start_pause="pause"), + timeout_penalty = dict(timeout_penalty_end="wait", start_pause="pause", end_state=True), + hold_penalty = dict(hold_penalty_end="wait", start_pause="pause", end_state=True), + delay_penalty = dict(delay_penalty_end="wait", start_pause="pause", end_state=True), + fixation_penalty = dict(fixation_penalty_end="wait", start_pause="pause", end_state=True), + reward = dict(reward_end="wait", start_pause="pause", stoppable=False, end_state=True), + pause = dict(end_pause="wait", end_state=True), + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Instantiate the targets + instantiate_targets = kwargs.pop('instantiate_targets', True) + if instantiate_targets: + + # Target 1 and 2 are for eye targets. Target 3 and 4 is for hand targets + target1 = VirtualRectangularTarget(target_width=self.fixation_radius, target_height=self.fixation_radius/2, target_color=target_colors[self.eye_target_color]) + target2 = VirtualRectangularTarget(target_width=self.fixation_radius, target_height=self.fixation_radius/2, target_color=target_colors[self.eye_target_color]) + target3 = VirtualCircularTarget(target_radius=self.target_radius, target_color=target_colors[self.target_color]) + target4 = VirtualCircularTarget(target_radius=self.target_radius, target_color=target_colors[self.target_color]) + + self.targets_eye = [target1, target2] + self.targets_hand = [target3, target4] + + self.offset_cube = np.array([0,10,self.fixation_radius/2]) # To center the cube target + + def _test_gaze_enter_target(self,ts): + ''' + Check whether eye positions and hand cursor are within the target radius + ''' + eye_pos = self.calibrated_eye_pos + eye_d = np.linalg.norm(eye_pos - self.eye_targs[self.eye_target_index,[0,2]]) + + cursor_pos = self.plant.get_endpoint_pos() + hand_d = np.linalg.norm(cursor_pos - self.hand_targs[self.hand_target_index]) + + return (eye_d <= self.target_radius + self.fixation_radius_buffer) and (hand_d <= self.target_radius - self.cursor_radius) + + def _test_gaze_target(self,ts): + ''' + Check whether eye positions are within the fixation distance + ''' + eye_pos = self.calibrated_eye_pos + eye_d = np.linalg.norm(eye_pos - self.eye_targs[self.eye_target_index,[0,2]]) + return eye_d <= self.target_radius + self.fixation_radius_buffer + + def _test_leave_target(self, ts): + ''' + check whether the hand cursor is outside the target distance + ''' + cursor_pos = self.plant.get_endpoint_pos() + hand_d = np.linalg.norm(cursor_pos - self.hand_targs[self.hand_target_index]) + return hand_d > (self.target_radius - self.cursor_radius) + + def _test_enter_target(self, ts): + ''' + check whether the hand cursor is within the target distance + ''' + cursor_pos = self.plant.get_endpoint_pos() + hand_d = np.linalg.norm(cursor_pos - self.hand_targs[self.hand_target_index]) + return hand_d <= (self.target_radius - self.cursor_radius) + + def _test_fixation_break(self,ts): + ''' + Triggers the fixation_penalty state when eye positions are outside fixation distance + Only apply this to the first hold and delay period + ''' + eye_pos = self.calibrated_eye_pos + eye_d = np.linalg.norm(eye_pos - self.eye_targs[self.eye_target_index,[0,2]]) + return eye_d > (self.target_radius + self.fixation_radius_buffer) + + def _test_fixation_complete(self, ts): + return ts > self.fixation_time + + def _test_hold_complete(self, ts): + return ts > self.hold_time + + def _test_delay_complete(self, ts): + ''' + Test whether the delay period, when the cursor or eye or both must stay in place + while another target is being presented, is over. + ''' + if self.target_index == 0: + return ts > self.delay_time1 + elif self.target_index == 1: + return ts > self.delay_time2 + else: + return True + + def _test_trial_complete(self, ts): + return self.eye_target_index == 1 and self.hand_target_index == 1 + + def _test_targ_simultaneous(self,ts): + return self.eye_target_index == 0 and self.hand_target_index == 0 and self.is_simultaneous_trials + + def _test_targ_first_sequence(self, ts): + return self.eye_target_index == 0 and self.hand_target_index == 0 and self.is_sequence_trials + + def _test_targ_second_sequence(self, ts): + return self.eye_target_index == 1 and self.hand_target_index == 0 and self.is_sequence_trials + + def _start_wait(self): + super()._start_wait() + + # Initialize target index and target positons for eye and hand + self.eye_target_index = -1 + self.hand_target_index = -1 + self.eye_targs = np.copy(self.targs) + self.hand_targs = np.copy(self.targs) + self.eye_gen_indices = np.copy(self.gen_indices) + self.hand_gen_indices = np.copy(self.gen_indices) + + if self.calc_trial_num() == 0: + + # Instantiate the targets here so they don't show up in any states that might come before "wait" + for target in self.targets_eye: + for model in target.graphics_models: + self.add_model(model) + target.hide() + + for target in self.targets_hand: + for model in target.graphics_models: + self.add_model(model) + target.hide() + + if self.tries == 0: # Update delay_time only in the first attempt + + # Set delay time + s, e = self.rand_delay1 + self.delay_time1 = random.random()*(e-s) + s + s, e = self.rand_delay2 + self.delay_time2 = random.random()*(e-s) + s + + # Decide sequence or simultaneous trials + self.is_sequence_trials = False + self.is_simultaneous_trials = False + + a = random.random() + if a < self.sequence_ratio: + self.is_sequence_trials = True + self.chain_length = 3 + else: + self.is_simultaneous_trials = True + self.chain_length = 2 + print(f'is sequence trials?: {self.is_sequence_trials}') + + def _start_target(self): + + self.target_index += 1 + self.eye_target_index += 1 + self.hand_target_index += 1 + + # Show eye hand target + target_hand = self.targets_hand[self.hand_target_index] + target_hand.move_to_position(self.hand_targs[self.hand_target_index]) + target_eye = self.targets_eye[self.eye_target_index] + target_eye.move_to_position(self.eye_targs[self.eye_target_index] - self.offset_cube) + target_hand.show() + target_eye.show() + self.sync_event('EYE_TARGET_ON', self.eye_gen_indices[self.eye_target_index]) + + def _while_target(self): + eye_pos = self.calibrated_eye_pos + eye_d = np.linalg.norm(eye_pos - self.eye_targs[self.eye_target_index,[0,2]]) + if eye_d <= (self.target_radius + self.fixation_radius_buffer): + self.targets_eye[self.eye_target_index].cube.color = target_colors[self.fixation_target_color] # chnage color in fixating center + else: + self.targets_eye[self.eye_target_index].cube.color = target_colors[self.eye_target_color] + + def _start_target_eye(self): + self.target_index += 1 + self.eye_target_index += 1 + + def _start_target_hand(self): + self.target_index += 1 + self.hand_target_index += 1 + + def _start_target_eye_hand(self): + self.target_index += 1 + self.eye_target_index += 1 + self.hand_target_index += 1 + + def _while_target_eye_hand(self): + eye_pos = self.calibrated_eye_pos + eye_d = np.linalg.norm(eye_pos - self.eye_targs[self.eye_target_index,[0,2]]) + if eye_d <= (self.target_radius + self.fixation_radius_buffer): + self.targets_eye[self.eye_target_index].cube.color = target_colors[self.fixation_target_color] # chnage color in fixating center + else: + self.targets_eye[self.eye_target_index].cube.color = target_colors[self.eye_target_color] + + def _start_fixation(self): + if self.target_index != 0: + self.sync_event('FIXATION', self.eye_gen_indices[self.eye_target_index]) + self.targets_eye[self.eye_target_index].cube.color = target_colors[self.fixation_target_color] + + def _start_hold(self): + if self.target_index != 1: # when the state comes from target_eye, skip start_hold + self.sync_event('CURSOR_ENTER_TARGET', self.hand_gen_indices[self.hand_target_index]) + + def _start_delay(self): + if self.target_index == 0 and self.is_simultaneous_trials: # This is for both eye and hand targets + next_idx = (self.eye_target_index + 1) + self.targets_eye[next_idx].move_to_position(self.eye_targs[next_idx] - self.offset_cube) + self.targets_hand[next_idx].move_to_position(self.hand_targs[next_idx]) + self.targets_eye[next_idx].show() + self.targets_hand[next_idx].show() + self.sync_event('EYE_TARGET_ON', self.eye_gen_indices[next_idx]) + + elif self.target_index == 0 and self.is_sequence_trials: # This is for eye target in the first delay + next_idx = (self.eye_target_index + 1) + self.targets_eye[next_idx].move_to_position(self.eye_targs[next_idx] - self.offset_cube) + self.targets_eye[next_idx].show() + self.sync_event('EYE_TARGET_ON', self.eye_gen_indices[next_idx]) + + elif self.target_index == 1 and self.is_sequence_trials: # This is for hand target in the second delay + next_idx = (self.hand_target_index + 1) + self.targets_hand[next_idx].move_to_position(self.hand_targs[next_idx]) # Target position is the same, but change color? + self.targets_hand[next_idx].show() + self.sync_event('TARGET_ON', self.hand_gen_indices[next_idx]) + + def _start_targ_transition(self): + if self.target_index == 0 and self.is_simultaneous_trials: # This is a go cue for both eye and hand + self.targets_eye[self.eye_target_index].hide() + self.targets_hand[self.hand_target_index].hide() + self.sync_event('EYE_TARGET_OFF', self.eye_gen_indices[self.eye_target_index]) + + elif self.target_index == 0 and self.is_sequence_trials: # This is a go cue for eye + self.targets_eye[self.eye_target_index].hide() + self.sync_event('EYE_TARGET_OFF', self.eye_gen_indices[self.eye_target_index]) + + elif self.target_index == 1 and self.is_sequence_trials: # This is a go cue for hand + self.targets_hand[self.hand_target_index].hide() + self.sync_event('TARGET_OFF', self.hand_gen_indices[self.hand_target_index]) + + def _start_timeout_penalty(self): + super()._start_timeout_penalty() + for target_eye, target_hand in zip(self.targets_eye,self.targets_hand): + target_eye.hide() + target_eye.reset() + target_hand.hide() + target_hand.reset() + + def _start_hold_penalty(self): + super()._start_hold_penalty() + for target_eye, target_hand in zip(self.targets_eye,self.targets_hand): + target_eye.hide() + target_eye.reset() + target_hand.hide() + target_hand.reset() + + def _start_delay_penalty(self): + super()._start_delay_penalty() + for target_eye, target_hand in zip(self.targets_eye,self.targets_hand): + target_eye.hide() + target_eye.reset() + target_hand.hide() + target_hand.reset() + + def _start_fixation_penalty(self): + super()._start_fixation_penalty() + for target_eye, target_hand in zip(self.targets_eye,self.targets_hand): + target_eye.hide() + target_eye.reset() + target_hand.hide() + target_hand.reset() + + def _start_reward(self): + super()._start_reward() + for target in self.targets_eye: + target.cue_trial_end_success() + + def _end_reward(self): + super()._end_reward() + for target_eye, target_hand in zip(self.targets_eye,self.targets_hand): + target_eye.hide() + target_eye.reset() + target_hand.hide() + target_hand.reset() + + def _start_pause(self): + super()._start_pause() + for target_eye, target_hand in zip(self.targets_eye,self.targets_hand): + target_eye.hide() + target_eye.reset() + target_hand.hide() + target_hand.reset() + class ScreenTargetCapture_Saccade(ScreenTargetCapture): ''' Center-out saccade task. The controller for the cursor position is eye position. @@ -414,8 +936,23 @@ class ScreenTargetCapture_Saccade(ScreenTargetCapture): ''' fixation_radius_buffer = traits.Float(.5, desc="additional radius for eye target") - target_color = traits.OptionsList("white", *target_colors, desc="Color of the target", bmi3d_input_options=list(target_colors.keys())) - fixation_target_color = traits.OptionsList("cyan", *target_colors, desc="Color of the eye target under fixation state", bmi3d_input_options=list(target_colors.keys())) + target_color = traits.OptionsList("eye_color", *target_colors, desc="Color of the target", bmi3d_input_options=list(target_colors.keys())) + fixation_target_color = traits.OptionsList("fixation_color", *target_colors, desc="Color of the eye target under fixation state", bmi3d_input_options=list(target_colors.keys())) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Instantiate the targets + instantiate_targets = kwargs.pop('instantiate_targets', True) + if instantiate_targets: + + # 2 targets for delay + target1 = VirtualRectangularTarget(target_width=self.target_radius, target_height=self.target_radius/2, target_color=target_colors[self.target_color]) + target2 = VirtualRectangularTarget(target_width=self.target_radius, target_height=self.target_radius/2, target_color=target_colors[self.target_color]) + + self.targets = [target1, target2] + + self.offset_cube = np.array([0,0,self.target_radius/2]) # To center the cube target def _test_enter_target(self, ts): ''' @@ -436,7 +973,30 @@ def _test_leave_target(self, ts): target_pos = np.delete(self.targs[self.target_index],1) d_eye = np.linalg.norm(eye_pos - target_pos) return (d_eye > self.target_radius + self.fixation_radius_buffer) or self.pause - + + def _start_target(self): + self.target_index += 1 + + # Show target if it is hidden (this is the first target, or previous state was a penalty) + target = self.targets[self.target_index % 2] + if self.target_index == 0: + target.move_to_position(self.targs[self.target_index] - self.offset_cube) + target.show() + self.sync_event('TARGET_ON', self.gen_indices[self.target_index]) + self.target_location = self.targs[self.target_index] # save for BMILoop + def _start_hold(self): super()._start_hold() - self.targets[self.target_index].sphere.color = target_colors[self.fixation_target_color] # change target color in fixating the target \ No newline at end of file + self.targets[self.target_index].cube.color = target_colors[self.fixation_target_color] # change target color in fixating the target + + def _start_delay(self): + # Make next target visible unless this is the final target in the trial + next_idx = (self.target_index + 1) + if next_idx < self.chain_length: + target = self.targets[next_idx % 2] + target.move_to_position(self.targs[next_idx] - self.offset_cube) + target.show() + self.sync_event('TARGET_ON', self.gen_indices[next_idx]) + else: + # This delay state should only last 1 cycle, don't sync anything + pass \ No newline at end of file diff --git a/built_in_tasks/target_graphics.py b/built_in_tasks/target_graphics.py index 3a78e496..36211ed1 100644 --- a/built_in_tasks/target_graphics.py +++ b/built_in_tasks/target_graphics.py @@ -36,6 +36,8 @@ "gold": (0.941,0.637,0.25,0.75), "elephant":(0.5,0.5,0.5,0.5), "white": (1, 1, 1, 0.75), + "eye_color": (0.85, 0.85, 0.85, 0.75), + "fixation_color": (0., 0.6, 0.6, 0.75), } class CircularTarget(object): @@ -145,18 +147,19 @@ def rotate_zaxis(self, angle, reset=False): self.cube.rotate_z(angle, reset=reset) def cue_trial_start(self): - self.cube.color = RED - self.show() + #self.cube.color = RED + #self.show() + pass def cue_trial_end_success(self): self.cube.color = GREEN def cue_trial_end_failure(self): - self.cube.color = YELLOW - self.hide() + self.cube.color = RED + #self.hide() def idle(self): - self.cube.color = RED + #self.cube.color = RED self.hide() def pt_inside(self, pt): diff --git a/features/eyetracker_features.py b/features/eyetracker_features.py index a3479231..e554d548 100644 --- a/features/eyetracker_features.py +++ b/features/eyetracker_features.py @@ -98,7 +98,7 @@ def get_target_locations(data, target_indices): print("Calibration complete:", self.eye_coeff) # Set up eye cursor - self.eye_cursor = VirtualCircularTarget(target_radius=.5, target_color=(0., 1., 0., 0.75)) + self.eye_cursor = VirtualCircularTarget(target_radius=.25, target_color=(0., 1., 0., 0.5)) self.target_location = np.array(self.starting_pos).copy() self.calibrated_eye_pos = np.zeros((2,))*np.nan for model in self.eye_cursor.graphics_models: diff --git a/features/generator_features.py b/features/generator_features.py index 811b30dc..e25ae41a 100644 --- a/features/generator_features.py +++ b/features/generator_features.py @@ -189,10 +189,17 @@ def _start_delay(self): next_idx = (self.target_index + 1) if next_idx < self.chain_length: target = self.targets[next_idx % 2] - self._old_target_color = np.copy(target.sphere.color) - new_target_color = list(target.sphere.color) - new_target_color[3] = self.delay_target_alpha - target.sphere.color = tuple(new_target_color) + + if hasattr(target, "sphere"): + self._old_target_color = np.copy(target.sphere.color) + new_target_color = list(target.sphere.color) + new_target_color[3] = self.delay_target_alpha + target.sphere.color = tuple(new_target_color) + elif hasattr(target, "cube"): + self._old_target_color = np.copy(target.cube.color) + new_target_color = list(target.cube.color) + new_target_color[3] = self.delay_target_alpha + target.cube.color = tuple(new_target_color) def _start_target(self): super()._start_target() @@ -200,7 +207,10 @@ def _start_target(self): # Reset the transparency of the current target if self.target_index > 0: target = self.targets[self.target_index % 2] - target.sphere.color = self._old_target_color + if hasattr(target, "sphere"): + target.sphere.color = self._old_target_color + elif hasattr(target, "cube"): + target.cube.color = self._old_target_color class StartTrialBelowSpeedThr(traits.HasTraits): '''