main <- dev

app.py

@@ -17,7 +17,10 @@
                   postProcessedDuration, makeRequestWithRetry,
                   writeToErrorLog)
 
-logging.basicConfig(level=logging.INFO)
+logging.basicConfig(format="[%(asctime)s] [%(levelname)s] %(message)s",
+                    level=logging.INFO,
+                    datefmt='%Y-%m-%d %H:%M:%S',
+                    force=True)
 
 API_TOKEN = getToken()
 API_URL = getAPIURL()
@@ -79,7 +82,8 @@
         continue
 
     if r.status_code == 404:
-        logging.info("...pulling " + workerType + " trials from " + API_URL)
+        logging.info(f"...pulling {workerType} trials from {API_URL} "
+                     f"using commit {getCommitHash()}")
         time.sleep(1)
 
         # When using autoscaling, we will remove the instance scale-in protection if it hasn't
@@ -124,10 +128,20 @@
         error_msg['error_msg'] = 'No videos uploaded. Ensure phones are connected and you have stable internet connection.'
         error_msg['error_msg_dev'] = 'No videos uploaded.'
 
-        r = makeRequestWithRetry('PATCH',
-                                 trial_url,
-                                 data={"status": "error", "meta": json.dumps(error_msg)},
-                                 headers = {"Authorization": "Token {}".format(API_TOKEN)})
+        try: 
+            r = makeRequestWithRetry('PATCH',
+                                    trial_url,
+                                    data={"status": "error", "meta": json.dumps(error_msg)},
+                                    headers = {"Authorization": "Token {}".format(API_TOKEN)})
+
+        except Exception as e:
+            traceback.print_exc()
+
+            if ERROR_LOG:
+                stack = traceback.format_exc()
+                writeToErrorLog(error_log_path, trial["session"], trial["id"],
+                                e, stack)
+
         continue
 
     # The following is now done in main, to allow reprocessing trials with missing videos

docker/docker-compose.yaml

@@ -38,6 +38,7 @@ services:
       options:
         max-size: "100m"  # Rotate when the log reaches 10MB
         max-file: "7"    # Keep the last 7 log files
+    restart: on-failure:3
   mmpose:
     image: ${MMPOSE_IMAGE_NAME}
     volumes:
@@ -55,6 +56,7 @@ services:
       options:
         max-size: "100m"  # Rotate when the log reaches 10MB
         max-file: "7"    # Keep the last 7 log files
+    restart: on-failure:3
 
 volumes:
   data: {}

main.py

@@ -603,7 +603,8 @@ def main(sessionName, trialName, trial_id, cameras_to_use=['all'],
                                          trialName + '.json')
         generateVisualizerJson(pathModelIK, pathOutputIK,
                                outputJsonVisPath, 
-                               vertical_offset=vertical_offset)  
+                               vertical_offset=vertical_offset,
+                               roundToRotations=4, roundToTranslations=4)
 
     # %% Rewrite settings, adding offset  
     if not extrinsicsTrial:

mmpose/loop_mmpose.py

@@ -69,8 +69,8 @@ def checkCudaPyTorch():
         if os.path.isfile(bboxPath):
             os.remove(bboxPath)
 
-        logging.info("Done. Cleaning up")
+        logging.info("mmpose: Done. Cleaning up")
 
     except:
-        logging.info("Pose detection failed.")
+        logging.info("mmpose: Pose detection failed.")
         os.remove(video_path)

openpose/loop_openpose.py

@@ -92,7 +92,7 @@ def getResolutionCommand(resolutionPoseDetection, horizontal):
         time.sleep(0.1)
         continue
 
-    logging.info("Processing...")
+    logging.info("Processing openpose...")
 
     if os.path.isdir(output_dir):
         shutil.rmtree(output_dir)
@@ -101,13 +101,18 @@ def getResolutionCommand(resolutionPoseDetection, horizontal):
     horizontal = getVideoOrientation(video_path)
     cmd_hr = getResolutionCommand(resolutionPoseDetection, horizontal)
 
-    check_cuda_device()
-    command = "/openpose/build/examples/openpose/openpose.bin\
-        --video {video_path}\
-        --display 0\
-        --write_json {output_dir}\
-        --render_pose 0{cmd_hr}".format(video_path=video_path, output_dir=output_dir, cmd_hr=cmd_hr)
-    os.system(command)
+    try: 
+        check_cuda_device()
+        command = "/openpose/build/examples/openpose/openpose.bin\
+            --video {video_path}\
+            --display 0\
+            --write_json {output_dir}\
+            --render_pose 0{cmd_hr}".format(video_path=video_path, output_dir=output_dir, cmd_hr=cmd_hr)
+        os.system(command)
 
-    logging.info("Done. Cleaning up")
-    os.remove(video_path)
+        logging.info("openpose: Done. Cleaning up")
+        os.remove(video_path)
+
+    except:
+        logging.info("openpose: Pose detection failed.")
+        os.remove(video_path)

utils.py

@@ -1586,7 +1586,7 @@ def checkCudaTF():
         sendStatusEmail(message=message)
         raise Exception("No GPU detected. Exiting.")
 
-def writeToJsonLog(path, new_dict, max_entries=1000):
+def writeToJsonLog(path, new_dict, max_entries=1000, indent=2):
     dir_name = os.path.dirname(path)
     if not os.path.exists(dir_name):
         os.makedirs(dir_name)
@@ -1603,7 +1603,7 @@ def writeToJsonLog(path, new_dict, max_entries=1000):
         data.pop(0)
 
     with open(path, 'w') as f:
-        json.dump(data, f)
+        json.dump(data, f, indent=indent)
 
 def writeToErrorLog(path, session_id, trial_id, error, stack, max_entries=1000):
     error_entry = {

utilsOpenSim.py

@@ -596,7 +596,7 @@ def compareTRCAndForcesTime(pathTRC,pathForces):
 # %% This takes model and IK and generates a json of body transforms that can 
 # be passed to the webapp visualizer
 def generateVisualizerJson(modelPath,ikPath,jsonOutputPath,statesInDegrees=True,
-                           vertical_offset=None):
+                           vertical_offset=None, roundToRotations=None, roundToTranslations=None):
 
     opensim.Logger.setLevelString('error')
     model = opensim.Model(modelPath)
@@ -693,8 +693,14 @@ def generateVisualizerJson(modelPath,ikPath,jsonOutputPath,statesInDegrees=True,
             # geometry origin. Ayman said getting transform to Geometry::Mesh is safest
             # but we don't have access to it thru API and Ayman said what we're doing
             # is OK for now
-            visualizeDict['bodies'][body.getName()]['rotation'].append(body.getTransformInGround(state).R().convertRotationToBodyFixedXYZ().to_numpy().tolist())
-            visualizeDict['bodies'][body.getName()]['translation'].append(body.getTransformInGround(state).T().to_numpy().tolist())
+            c_rotations = body.getTransformInGround(state).R().convertRotationToBodyFixedXYZ().to_numpy()
+            c_translations = body.getTransformInGround(state).T().to_numpy()
+            if roundToRotations is not None:                
+                c_rotations = np.round(c_rotations, roundToRotations)
+            if roundToTranslations is not None:
+                c_translations = np.round(c_translations, roundToTranslations)
+            visualizeDict['bodies'][body.getName()]['rotation'].append(c_rotations.tolist())
+            visualizeDict['bodies'][body.getName()]['translation'].append(c_translations.tolist())            
 
     with open(jsonOutputPath, 'w') as f:
         json.dump(visualizeDict, f)

utilsServer.py

@@ -6,6 +6,7 @@
 import logging
 import time
 import random
+import urllib
 
 from main import main
 from utils import getDataDirectory
@@ -500,28 +501,28 @@ def runTestSession(pose='all',isDocker=True,maxNumTries=3):
             logging.info("\n\n\nStatus check succeeded. \n\n")
             return
 
-        # Catch and re-enter while loop if it's an HTTPError (could be more
-        # than just 404 errors). Wait between 30 and 60 seconds before 
-        # retrying.
-        except requests.exceptions.HTTPError as e:
+        # Catch and re-enter while loop if it's an HTTPError or URLError 
+        # (could be more than just 404 errors). Wait between 30 and 60 seconds 
+        # before retrying.
+        except (requests.exceptions.HTTPError, urllib.error.URLError) as e:
             if numTries < maxNumTries:
-                logging.info(f"test trial failed on try #{numTries} due to HTTPError. Retrying.")
+                logging.info(f"test trial failed on try #{numTries} due to HTTPError or URLError. Retrying.")
                 wait_time = random.randint(30,60)
                 logging.info(f"waiting {wait_time} seconds then retrying...")
                 time.sleep(wait_time)
                 continue
             else:
-                logging.info(f"test trial failed on try #{numTries} due to HTTPError.")
+                logging.info(f"test trial failed on try #{numTries} due to HTTPError or URLError.")
                 # send email
-                message = "A backend OpenCap machine failed the status check (HTTPError). It has been stopped."
+                message = "A backend OpenCap machine failed the status check (HTTPError or URLError). It has been stopped."
                 sendStatusEmail(message=message)
-                raise Exception('Failed status check (HTTPError). Stopped.')
+                raise Exception('Failed status check (HTTPError or URLError). Stopped.')
 
         # Catch other errors and stop
         except:
             logging.info("test trial failed. stopping machine.")
             # send email
-            message = "A backend OpenCap machine failed the status check. It has been stopped."
+            message = "A backend OpenCap machine failed the status check (not HTTPError or URLError). It has been stopped."
             sendStatusEmail(message=message)
             raise Exception('Failed status check. Stopped.')