From 6b17340dd5bd98e27deb33f52e335791f6f69faf Mon Sep 17 00:00:00 2001 From: albertna Date: Wed, 30 Aug 2023 11:40:39 -0500 Subject: [PATCH 1/8] Implement G3 node --- chimerapy/pipelines/g3/__init__.py | 0 chimerapy/pipelines/g3/node.py | 154 +++++++++++++++++++++++++++++ pyproject.toml | 24 +++-- 3 files changed, 165 insertions(+), 13 deletions(-) create mode 100644 chimerapy/pipelines/g3/__init__.py create mode 100644 chimerapy/pipelines/g3/node.py diff --git a/chimerapy/pipelines/g3/__init__.py b/chimerapy/pipelines/g3/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/chimerapy/pipelines/g3/node.py b/chimerapy/pipelines/g3/node.py new file mode 100644 index 0000000..531128f --- /dev/null +++ b/chimerapy/pipelines/g3/node.py @@ -0,0 +1,154 @@ +from typing import List, Optional + +import asyncio +import cv2 +import logging + +from g3pylib import connect_to_glasses + +import chimerapy.engine as cpe +from chimerapy.orchestrator import source_node + + +@source_node(name="CPPipelines_G3") +class G3(cpe.Node): + """A node that connects to a G3 instance to perform streaming and recording operations. + + Sample `gaze_data` in datachunk: + ```JSON + { + "gaze2d": [0.468, 0.483], + "gaze3d": [37.543, -18.034, 821.265], + "eyeleft": { + "gazeorigin": [28.799, -7.165, -23.945], + "gazedirection": [0.0463, -0.0337, 0.998], + "pupildiameter": 2.633 + }, + "eyeright": { + "gazeorigin": [-28.367, -5.353, -21.426], + "gazedirection": [0.0436, 0.00611, 0.999], + "pupildiameter": 2.782 + } + } + ``` + + Parameters + ---------- + hostname: str, required + The key to use for the frame in the data chunk + name: str, optional (default: "") + The name of the node. If not provided, the hostname will be used. + show_gaze: bool, optional (default: False) + Whether to render the gaze circle on video frames + frames_key: str, optional (default: "frame") + The key to use for the video frame in the data chunk + **kwargs + Additional keyword arguments to pass to the Node constructor + """ + + def __init__( + self, + hostname: str, + name: str = "", + show_gaze: bool = False, + frame_key: str = "frame", + **kwargs, + ): + self.hostname = hostname + self.show_gaze = show_gaze + self.frame_key = frame_key + + if not name: + name = hostname + + super().__init__(name=name, **kwargs) + + async def setup(self) -> None: + self.g3 = await connect_to_glasses.with_hostname( + self.hostname, using_zeroconf=True + ) + + streams = await self.g3.stream_rtsp(scene_camera=True, gaze=True) + self.scene_stream = streams.scene_camera.decode() + self.gaze_stream = streams.gaze.decode() + + async def step(self) -> cpe.DataChunk: + ret_chunk = cpe.DataChunk() + + frame_data, frame_timestamp = await self.scene_stream.get() + gaze_data, gaze_timestamp = await self.gaze_stream.get() + + # Match frame and gaze timestamps + while gaze_timestamp is None or frame_timestamp is None: + if frame_timestamp is None: + frame_data, frame_timestamp = await self.scene_stream.get() + if gaze_timestamp is None: + gaze_data, gaze_timestamp = await self.gaze_stream.get() + while gaze_timestamp < frame_timestamp: + gaze_data, gaze_timestamp = await self.gaze_stream.get() + while gaze_timestamp is None: + gaze_data, gaze_timestamp = await self.gaze_stream.get() + + # logging.info(f"Frame timestamp: {frame_timestamp}") + # logging.info(f"Gaze timestamp: {gaze_timestamp}") + frame_data = frame_data.to_ndarray(format="bgr24") + + if self.show_gaze and "gaze2d" in gaze_data: + gaze2d = gaze_data["gaze2d"] + logging.info(f"Gaze2d: {gaze2d[0]:9.4f},{gaze2d[1]:9.4f}") + + # Convert rational (x,y) to pixel location (x,y) + h, w = frame_data.shape[:2] + fix = (int(gaze2d[0] * w), int(gaze2d[1] * h)) + + # Draw gaze + frame_data = cv2.circle(frame_data, fix, 10, (0, 0, 255), 3) + + ret_chunk.add(self.frame_key, frame_data, "image") + ret_chunk.add("gaze_data", gaze_data) + ret_chunk.add("frame_timestamp", frame_timestamp) + ret_chunk.add("gaze_timestamp", gaze_timestamp) + + return ret_chunk + + @cpe.register # .with_config(style="blocking") + async def calibrate(self) -> bool: + return await self.g3.calibrate.run() + + # TODO: check recorder state before making start, stop, cancel calls + # TODO: add visual cue to indicate recorder state, e.g. UI components or greyed out buttons + @cpe.register + async def start_recording(self) -> bool: + return await self.g3.recorder.start() + + @cpe.register + async def cancel_recording(self) -> None: + await self.g3.recorder.cancel() + + @cpe.register + async def stop_recording(self) -> bool: + # TODO: check if recording has started + return await self.g3.recorder.stop() + + @cpe.register + async def take_snapshot(self) -> bool: + return await self.g3.recorder.snapshot() + + @cpe.register.with_config(params={"download_dir": "str"}) + async def download_latest_recording(self, download_dir: str) -> Optional[str]: + if not await self.g3.recordings: + return None + + return await self.g3.recordings[0].download_files(download_dir) + + @cpe.register.with_config(params={"download_dir": "str"}) + async def download_all_recordings(self, download_dir: str) -> Optional[List[str]]: + if not await self.g3.recordings: + return None + + return await asyncio.gather( + recording.download_files(download_dir) for recording in self.g3.recordings + ) + + def teardown(self) -> None: + self.g3.close() diff --git a/pyproject.toml b/pyproject.toml index b1e3030..72799f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,15 +2,13 @@ name = "chimerapy-pipelines" version = "0.0.1" description = "Respository of sharable pipelines for ChimeraPy" -license = {file = "LICENSE.txt"} +license = { file = "LICENSE.txt" } readme = "README.md" requires-python = ">3.6" keywords = ["education", "multimodal", "data", "learning", "analytics"] -classifiers = [ - "Programming Language :: Python :: 3" -] +classifiers = ["Programming Language :: Python :: 3"] dependencies = [ 'numpy', @@ -24,7 +22,7 @@ dependencies = [ 'requests', 'mss', 'chimerapy-orchestrator', - 'chimerapy-engine' + 'chimerapy-engine', ] [project.optional-dependencies] @@ -38,16 +36,18 @@ test = [ mfsort = [ 'mf_sort[yolo] @ git+https://github.com/kbvatral/MF-SORT.git@master#egg=mf_sort', - 'ultralytics' + 'ultralytics', ] embodied = [ 'elp @ git+http://github.com/oele-isis-vanderbilt/EmbodiedLearningProject@main#egg=elp', - 'l2cs @ git+https://github.com/edavalosanaya/L2CS-Net.git@main#egg=l2cs' + 'l2cs @ git+https://github.com/edavalosanaya/L2CS-Net.git@main#egg=l2cs', ] -yolov8 = [ - 'ultralytics' +yolov8 = ['ultralytics'] + +g3 = [ + 'g3pylib @ git+https://github.com/albertna/g3pylib.git@feature/download-recording#egg=g3pylib', ] [project.urls] @@ -70,7 +70,7 @@ where = ["."] ignore = ["E501"] select = ["E", "W", "F", "C", "B", "I"] ignore-init-module-imports = true -fixable = ["I001"] # isort fix only +fixable = ["I001"] # isort fix only extend-exclude = ["run.py"] [tool.ruff.per-file-ignores] @@ -78,8 +78,6 @@ extend-exclude = ["run.py"] "chimerapy/pipelines/__version__.py" = ["E402"] - - [project.entry-points."chimerapy.orchestrator.nodes_registry"] get_nodes_registry = "chimerapy.pipelines:register_nodes_metadata" @@ -94,7 +92,7 @@ log_cli_format = "%(asctime)s.%(msecs)03d [%(levelname)8s] %(message)s (%(filena log_cli_date_format = "%Y-%m-%d %H:%M:%S" # Timeout -faulthandler_timeout=300 +faulthandler_timeout = 300 # Ignore warnings filterwarnings = "ignore::DeprecationWarning" From 65c09bee61c34148f9f144921e390183e9679706 Mon Sep 17 00:00:00 2001 From: albertna Date: Thu, 31 Aug 2023 17:09:57 -0500 Subject: [PATCH 2/8] Update docstring --- chimerapy/pipelines/g3/node.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/chimerapy/pipelines/g3/node.py b/chimerapy/pipelines/g3/node.py index 531128f..f8f6208 100644 --- a/chimerapy/pipelines/g3/node.py +++ b/chimerapy/pipelines/g3/node.py @@ -12,7 +12,7 @@ @source_node(name="CPPipelines_G3") class G3(cpe.Node): - """A node that connects to a G3 instance to perform streaming and recording operations. + """A node that connects to a Tobii Glasses 3 instance to perform streaming and recording operations. Sample `gaze_data` in datachunk: ```JSON @@ -35,9 +35,9 @@ class G3(cpe.Node): Parameters ---------- hostname: str, required - The key to use for the frame in the data chunk + The G3 device's serial number (e.g. TG03B-080200004381), used for connections. name: str, optional (default: "") - The name of the node. If not provided, the hostname will be used. + The name of the node. If not provided, the `hostname` will be used. show_gaze: bool, optional (default: False) Whether to render the gaze circle on video frames frames_key: str, optional (default: "frame") @@ -106,8 +106,8 @@ async def step(self) -> cpe.DataChunk: ret_chunk.add(self.frame_key, frame_data, "image") ret_chunk.add("gaze_data", gaze_data) - ret_chunk.add("frame_timestamp", frame_timestamp) - ret_chunk.add("gaze_timestamp", gaze_timestamp) + # ret_chunk.add("frame_timestamp", frame_timestamp) + # ret_chunk.add("gaze_timestamp", gaze_timestamp) return ret_chunk From 3c61c79ddce27a305d4c73cefddb896088bdfbb4 Mon Sep 17 00:00:00 2001 From: albertna Date: Tue, 3 Oct 2023 11:35:32 -0500 Subject: [PATCH 3/8] Add G3 node registration to `__init__.py` --- .gitignore | 3 +++ chimerapy/pipelines/__init__.py | 1 + 2 files changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 19232cf..9b290ea 100644 --- a/.gitignore +++ b/.gitignore @@ -134,3 +134,6 @@ dmypy.json # Pyre type checker .pyre/ + +# IDE +.vscode/ diff --git a/chimerapy/pipelines/__init__.py b/chimerapy/pipelines/__init__.py index 77601d0..a0e7adf 100644 --- a/chimerapy/pipelines/__init__.py +++ b/chimerapy/pipelines/__init__.py @@ -53,6 +53,7 @@ def register_nodes_metadata(): "chimerapy.pipelines.yolov8.multi_vid_pose:YoloV8Node", "chimerapy.pipelines.yolov8.multi_save:MultiSaveNode", "chimerapy.pipelines.yolov8.display:DisplayNode", + "chimerapy.pipelines.g3.node:G3", ], } From 592e0c1252680133a6844a4d7d501ca7a0f5cd74 Mon Sep 17 00:00:00 2001 From: albertna Date: Tue, 3 Oct 2023 12:04:33 -0500 Subject: [PATCH 4/8] Add single g3 test config --- chimerapy/pipelines/g3/node.py | 2 ++ configs/g3/all_local_single_device.json | 46 +++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 configs/g3/all_local_single_device.json diff --git a/chimerapy/pipelines/g3/node.py b/chimerapy/pipelines/g3/node.py index f8f6208..b60a19f 100644 --- a/chimerapy/pipelines/g3/node.py +++ b/chimerapy/pipelines/g3/node.py @@ -152,3 +152,5 @@ async def download_all_recordings(self, download_dir: str) -> Optional[List[str] def teardown(self) -> None: self.g3.close() + self.scene_stream = None + self.gaze_stream = None diff --git a/configs/g3/all_local_single_device.json b/configs/g3/all_local_single_device.json new file mode 100644 index 0000000..90ca363 --- /dev/null +++ b/configs/g3/all_local_single_device.json @@ -0,0 +1,46 @@ +{ + "mode": "preview", + "workers": { + "manager_ip": "172.16.80.147", + "manager_port": 9001, + "instances": [ + { + "name": "local", + "id": "local", + "description": "local worker for the MMLA pipeline demo with a G3 node" + } + ] + }, + "nodes": [ + { + "registry_name": "CPPipelines_G3", + "name": "g3-instance", + "kwargs": { + "hostname": "TG03B-080201037351", + "show_gaze": true + }, + "package": "chimerapy-pipelines" + }, + { + "registry_name": "CPPipelines_ShowWindows", + "name": "show", + "kwargs": { + "window_xy": [1280, 720], + "items_per_row": 2 + }, + "package": "chimerapy-pipelines" + } + ], + "adj": [["g3-instance", "show"]], + "manager_config": { + "logdir": "cp-logs", + "port": 9001 + }, + "mappings": { + "local": ["g3-instance", "show"] + }, + "timeouts": { + "commit_timeout": 120, + "shutdown_timeout": 300 + } +} From e167556c8f572f75fce8af59338574bf57726ca0 Mon Sep 17 00:00:00 2001 From: albertna Date: Fri, 6 Oct 2023 13:41:11 -0500 Subject: [PATCH 5/8] Use rudimentary streams to avoid using context managers --- chimerapy/pipelines/g3/node.py | 58 +++++++++++++++---------- configs/g3/all_local_single_device.json | 4 -- 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/chimerapy/pipelines/g3/node.py b/chimerapy/pipelines/g3/node.py index b60a19f..07d1885 100644 --- a/chimerapy/pipelines/g3/node.py +++ b/chimerapy/pipelines/g3/node.py @@ -1,8 +1,10 @@ from typing import List, Optional import asyncio +import base64 import cv2 import logging +import numpy as np from g3pylib import connect_to_glasses @@ -68,30 +70,32 @@ async def setup(self) -> None: self.hostname, using_zeroconf=True ) - streams = await self.g3.stream_rtsp(scene_camera=True, gaze=True) - self.scene_stream = streams.scene_camera.decode() - self.gaze_stream = streams.gaze.decode() + ( + self.scene_queue, + self.unsub_to_scene, + ) = await self.g3.rudimentary.subscribe_to_scene() + ( + self.gaze_queue, + self.unsub_to_gaze, + ) = await self.g3.rudimentary.subscribe_to_gaze() + + await self.g3.rudimentary.start_streams() async def step(self) -> cpe.DataChunk: ret_chunk = cpe.DataChunk() - frame_data, frame_timestamp = await self.scene_stream.get() - gaze_data, gaze_timestamp = await self.gaze_stream.get() + frame_timestamp, frame_data_b64 = await self.scene_queue.get() + gaze_timestamp, gaze_data = await self.gaze_queue.get() + + # print("frame_data, typeof ", str(type(frame_data)), frame_data) + # print("frame_timestamp, typeof ", str(type(frame_timestamp)), frame_timestamp) + # print("gaze_data, typeof ", str(type(gaze_data)), gaze_data) + # print("gaze_timestamp, typeof ", str(type(gaze_timestamp)), gaze_timestamp) - # Match frame and gaze timestamps - while gaze_timestamp is None or frame_timestamp is None: - if frame_timestamp is None: - frame_data, frame_timestamp = await self.scene_stream.get() - if gaze_timestamp is None: - gaze_data, gaze_timestamp = await self.gaze_stream.get() - while gaze_timestamp < frame_timestamp: - gaze_data, gaze_timestamp = await self.gaze_stream.get() - while gaze_timestamp is None: - gaze_data, gaze_timestamp = await self.gaze_stream.get() + frame_nparr = np.fromstring(base64.b64decode(frame_data_b64), dtype=np.uint8) + frame_data = cv2.imdecode(frame_nparr, cv2.IMREAD_COLOR) - # logging.info(f"Frame timestamp: {frame_timestamp}") - # logging.info(f"Gaze timestamp: {gaze_timestamp}") - frame_data = frame_data.to_ndarray(format="bgr24") + # self.save_video("stream", frame_data, 25) if self.show_gaze and "gaze2d" in gaze_data: gaze2d = gaze_data["gaze2d"] @@ -104,15 +108,18 @@ async def step(self) -> cpe.DataChunk: # Draw gaze frame_data = cv2.circle(frame_data, fix, 10, (0, 0, 255), 3) + # TODO: figure out why CV2 window isn't showing ret_chunk.add(self.frame_key, frame_data, "image") ret_chunk.add("gaze_data", gaze_data) - # ret_chunk.add("frame_timestamp", frame_timestamp) - # ret_chunk.add("gaze_timestamp", gaze_timestamp) + # # ret_chunk.add("frame_timestamp", frame_timestamp) + # # ret_chunk.add("gaze_timestamp", gaze_timestamp) return ret_chunk @cpe.register # .with_config(style="blocking") async def calibrate(self) -> bool: + # TODO: test if calibration works separately for rudimentary and regular + await self.g3.rudimentary.calibrate() return await self.g3.calibrate.run() # TODO: check recorder state before making start, stop, cancel calls @@ -150,7 +157,10 @@ async def download_all_recordings(self, download_dir: str) -> Optional[List[str] recording.download_files(download_dir) for recording in self.g3.recordings ) - def teardown(self) -> None: - self.g3.close() - self.scene_stream = None - self.gaze_stream = None + async def teardown(self) -> None: + # await self.streams.__aexit__() + await self.g3.rudimentary.stop_streams() + await self.unsub_to_scene + await self.unsub_to_gaze + + await self.g3.close() diff --git a/configs/g3/all_local_single_device.json b/configs/g3/all_local_single_device.json index 90ca363..eb5098d 100644 --- a/configs/g3/all_local_single_device.json +++ b/configs/g3/all_local_single_device.json @@ -24,10 +24,6 @@ { "registry_name": "CPPipelines_ShowWindows", "name": "show", - "kwargs": { - "window_xy": [1280, 720], - "items_per_row": 2 - }, "package": "chimerapy-pipelines" } ], From d37461fdd2a46b368d9de9c2c8fd69f148c1b65c Mon Sep 17 00:00:00 2001 From: albertna Date: Wed, 11 Oct 2023 11:50:40 -0500 Subject: [PATCH 6/8] Start and stop on-device recording based on node state --- chimerapy/pipelines/g3/node.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/chimerapy/pipelines/g3/node.py b/chimerapy/pipelines/g3/node.py index 07d1885..353125a 100644 --- a/chimerapy/pipelines/g3/node.py +++ b/chimerapy/pipelines/g3/node.py @@ -80,8 +80,17 @@ async def setup(self) -> None: ) = await self.g3.rudimentary.subscribe_to_gaze() await self.g3.rudimentary.start_streams() + self.is_recording = False async def step(self) -> cpe.DataChunk: + # start on-device recording if node enters RECORDING state + if self.state.fsm == "RECORDING" and not self.is_recording: + await self.g3.recorder.start() + self.is_recording = True + elif self.state.fsm == "STOPPED" and self.is_recording: + await self.g3.recorder.stop() + self.is_recording = False + ret_chunk = cpe.DataChunk() frame_timestamp, frame_data_b64 = await self.scene_queue.get() @@ -110,7 +119,9 @@ async def step(self) -> cpe.DataChunk: # TODO: figure out why CV2 window isn't showing ret_chunk.add(self.frame_key, frame_data, "image") + # print("---------------------ADDED FRAME DATA------------------------") ret_chunk.add("gaze_data", gaze_data) + # print("---------------------ADDED GAZE DATA------------------------") # # ret_chunk.add("frame_timestamp", frame_timestamp) # # ret_chunk.add("gaze_timestamp", gaze_timestamp) @@ -124,18 +135,18 @@ async def calibrate(self) -> bool: # TODO: check recorder state before making start, stop, cancel calls # TODO: add visual cue to indicate recorder state, e.g. UI components or greyed out buttons - @cpe.register - async def start_recording(self) -> bool: - return await self.g3.recorder.start() + # @cpe.register + # async def start_recording(self) -> bool: + # return await self.g3.recorder.start() @cpe.register async def cancel_recording(self) -> None: await self.g3.recorder.cancel() - @cpe.register - async def stop_recording(self) -> bool: - # TODO: check if recording has started - return await self.g3.recorder.stop() + # @cpe.register + # async def stop_recording(self) -> bool: + # # TODO: check if recording has started + # return await self.g3.recorder.stop() @cpe.register async def take_snapshot(self) -> bool: @@ -159,6 +170,7 @@ async def download_all_recordings(self, download_dir: str) -> Optional[List[str] async def teardown(self) -> None: # await self.streams.__aexit__() + await self.g3.recorder.stop() await self.g3.rudimentary.stop_streams() await self.unsub_to_scene await self.unsub_to_gaze From f436566f9f5782f3d1008a3ba62ba574bb742c24 Mon Sep 17 00:00:00 2001 From: albertna Date: Fri, 27 Oct 2023 14:07:47 -0500 Subject: [PATCH 7/8] Print messages for starting/stopping on-device recordings --- chimerapy/pipelines/g3/node.py | 12 +++++++----- configs/g3/all_local_single_device.json | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/chimerapy/pipelines/g3/node.py b/chimerapy/pipelines/g3/node.py index 353125a..0c00331 100644 --- a/chimerapy/pipelines/g3/node.py +++ b/chimerapy/pipelines/g3/node.py @@ -85,11 +85,13 @@ async def setup(self) -> None: async def step(self) -> cpe.DataChunk: # start on-device recording if node enters RECORDING state if self.state.fsm == "RECORDING" and not self.is_recording: - await self.g3.recorder.start() - self.is_recording = True + if await self.g3.recorder.start(): + self.is_recording = True + print("\n---------------Recording Started On Device---------------") elif self.state.fsm == "STOPPED" and self.is_recording: - await self.g3.recorder.stop() - self.is_recording = False + if await self.g3.recorder.stop(): + self.is_recording = False + print("\n---------------Recording Stopped On Device---------------") ret_chunk = cpe.DataChunk() @@ -104,7 +106,7 @@ async def step(self) -> cpe.DataChunk: frame_nparr = np.fromstring(base64.b64decode(frame_data_b64), dtype=np.uint8) frame_data = cv2.imdecode(frame_nparr, cv2.IMREAD_COLOR) - # self.save_video("stream", frame_data, 25) + self.save_video("stream", frame_data, 25) if self.show_gaze and "gaze2d" in gaze_data: gaze2d = gaze_data["gaze2d"] diff --git a/configs/g3/all_local_single_device.json b/configs/g3/all_local_single_device.json index eb5098d..f9ce495 100644 --- a/configs/g3/all_local_single_device.json +++ b/configs/g3/all_local_single_device.json @@ -1,5 +1,5 @@ { - "mode": "preview", + "mode": "record", "workers": { "manager_ip": "172.16.80.147", "manager_port": 9001, From 8c852ddf919dc988adcde191cd3059ed6497db45 Mon Sep 17 00:00:00 2001 From: albertna Date: Fri, 27 Oct 2023 14:48:23 -0500 Subject: [PATCH 8/8] Cleanups and save gaze data to log file --- chimerapy/pipelines/g3/node.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/chimerapy/pipelines/g3/node.py b/chimerapy/pipelines/g3/node.py index 0c00331..f61be31 100644 --- a/chimerapy/pipelines/g3/node.py +++ b/chimerapy/pipelines/g3/node.py @@ -87,45 +87,45 @@ async def step(self) -> cpe.DataChunk: if self.state.fsm == "RECORDING" and not self.is_recording: if await self.g3.recorder.start(): self.is_recording = True - print("\n---------------Recording Started On Device---------------") + print("\n---------------Started Recording On Device---------------") elif self.state.fsm == "STOPPED" and self.is_recording: if await self.g3.recorder.stop(): self.is_recording = False - print("\n---------------Recording Stopped On Device---------------") + print("\n---------------Stopped Recording On Device---------------") ret_chunk = cpe.DataChunk() frame_timestamp, frame_data_b64 = await self.scene_queue.get() gaze_timestamp, gaze_data = await self.gaze_queue.get() - # print("frame_data, typeof ", str(type(frame_data)), frame_data) - # print("frame_timestamp, typeof ", str(type(frame_timestamp)), frame_timestamp) - # print("gaze_data, typeof ", str(type(gaze_data)), gaze_data) - # print("gaze_timestamp, typeof ", str(type(gaze_timestamp)), gaze_timestamp) + # add timestamp to gaze data + gaze_data["timestamp"] = gaze_timestamp + # convert base64-encoded frame data string to cv2 image frame_nparr = np.fromstring(base64.b64decode(frame_data_b64), dtype=np.uint8) frame_data = cv2.imdecode(frame_nparr, cv2.IMREAD_COLOR) - self.save_video("stream", frame_data, 25) - if self.show_gaze and "gaze2d" in gaze_data: gaze2d = gaze_data["gaze2d"] logging.info(f"Gaze2d: {gaze2d[0]:9.4f},{gaze2d[1]:9.4f}") - # Convert rational (x,y) to pixel location (x,y) + # convert rational (x,y) to pixel location (x,y) h, w = frame_data.shape[:2] fix = (int(gaze2d[0] * w), int(gaze2d[1] * h)) - # Draw gaze + # draw gaze on frame_data frame_data = cv2.circle(frame_data, fix, 10, (0, 0, 255), 3) + # save gaze_data to file + self.save_json("gaze-data", gaze_data) + + # self.save_video("stream", frame_data, 25) + # TODO: figure out why CV2 window isn't showing ret_chunk.add(self.frame_key, frame_data, "image") # print("---------------------ADDED FRAME DATA------------------------") ret_chunk.add("gaze_data", gaze_data) # print("---------------------ADDED GAZE DATA------------------------") - # # ret_chunk.add("frame_timestamp", frame_timestamp) - # # ret_chunk.add("gaze_timestamp", gaze_timestamp) return ret_chunk