From 6492775d16b279b362a93608e99b039ec35b866a Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Fri, 28 Nov 2025 12:58:09 +0000 Subject: [PATCH 01/18] Some of the changes from experimental --- .github/workflows/docker-image.yml | 10 +++++++--- .gitignore | 3 +++ run-jl.sh | 5 +++++ run.sh | 25 ++++++++++++++++++++----- 4 files changed, 35 insertions(+), 8 deletions(-) create mode 100755 run-jl.sh diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 950a116..d66ef0e 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -2,9 +2,9 @@ name: Docker Image Build and Run on: push: - branches: [ master, dev*, test* ] + branches: [ master, dev*, experimental, test* ] pull_request: - branches: [ master, dev*, test* ] + branches: [ master, dev*, experimental, test* ] jobs: @@ -23,8 +23,12 @@ jobs: run: | docker -v docker images - - name: Run the Docker image + - name: Run the Docker image with the falling cube run: | + ./run.sh -d 20 -c demo1 + - name: Run the Docker image with a worm + run: | + ./stop.sh ./run.sh - name: Info on generated files run: | diff --git a/.gitignore b/.gitignore index 676e1f4..0bea9b4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ output/C2* /output/siminfo.py /output/wor* /output/Sibernetic_* +/output/test* +/output/box +/output/swim* diff --git a/run-jl.sh b/run-jl.sh new file mode 100755 index 0000000..fb39de1 --- /dev/null +++ b/run-jl.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +version=$(&2 + *) echo "Usage: $0 [-d duration] [-c configuration]" >&2 + exit 1 ;; esac done @@ -15,7 +19,6 @@ done OW_OUT_DIR=/home/ow/shared HOST_OUT_DIR=$PWD -version=$( Date: Fri, 28 Nov 2025 12:58:22 +0000 Subject: [PATCH 02/18] Temp: run a longer sim... --- .github/workflows/docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index d66ef0e..7828989 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -29,7 +29,7 @@ jobs: - name: Run the Docker image with a worm run: | ./stop.sh - ./run.sh + ./run.sh -d 150 - name: Info on generated files run: | ls -alth From 2dea24776aad8cc8efd20dcfe1da845f9525a8dd Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Fri, 28 Nov 2025 18:15:43 +0000 Subject: [PATCH 03/18] Just copy position files out --- master_openworm.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/master_openworm.py b/master_openworm.py index a8344c0..b2b8273 100644 --- a/master_openworm.py +++ b/master_openworm.py @@ -237,6 +237,16 @@ def execute_with_realtime_output(command, directory, env=None): print("Moving %s to %s" % (report, new_sim_out)) shutil.move(report, new_sim_out) +# Copy position files etc. +txt_files = glob.glob("%s/*.txt" % latest_subdir) +for txt_file in txt_files: + print("Moving %s to %s" % (txt_file, new_sim_out)) + shutil.move(txt_file, new_sim_out) +dat_files = glob.glob("%s/*.dat" % latest_subdir) +for dat_file in dat_files: + print("Moving %s to %s" % (dat_file, new_sim_out)) + shutil.move(dat_file, new_sim_out) + # Copy WCON file(s) wcons = glob.glob("%s/*.wcon" % latest_subdir) for wcon in wcons: From 3fcb10e3521039cfcff08b6f4b464a30af17e768 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Thu, 4 Dec 2025 17:19:23 +0000 Subject: [PATCH 04/18] Changes from experimental --- .github/workflows/docker-image.yml | 16 +++++++++++++--- .gitignore | 3 +++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 7828989..2f9b2b3 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -14,22 +14,32 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Build the Docker image run: | chmod a+w output/ ls -alth ./build.sh + - name: Info on Docker images run: | docker -v docker images - - name: Run the Docker image with the falling cube + + - name: Run the Docker image with the default falling cube demo from sibernetic run: | ./run.sh -d 20 -c demo1 - - name: Run the Docker image with a worm + + - name: Run the Docker image with a worm simulation - no c302 run: | ./stop.sh - ./run.sh -d 150 + ./run.sh -d 10 -n + + - name: Run the Docker image with a worm simulation + run: | + ./stop.sh + ./run.sh -d 10 + - name: Info on generated files run: | ls -alth diff --git a/.gitignore b/.gitignore index 0bea9b4..b8e0b5e 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ output/C2* /output/test* /output/box /output/swim* +/output/alone* +/output/cube* +/output/master* From 446749b709cb3f6547e706a59701ab0ab33fc392 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Thu, 4 Dec 2025 17:38:25 +0000 Subject: [PATCH 05/18] Latest run sh and py --- master_openworm.py | 104 ++++++++++++++------------------------------- run.sh | 17 +++++++- 2 files changed, 48 insertions(+), 73 deletions(-) diff --git a/master_openworm.py b/master_openworm.py index b2b8273..5c12e5f 100644 --- a/master_openworm.py +++ b/master_openworm.py @@ -12,12 +12,10 @@ import math print("*****************************") -print("OpenWorm Master Script") +print(" OpenWorm Master Script") print("*****************************") print("") -print( - "This script attempts to run a full pass through the OpenWorm scientific libraries." -) +print("This script attempts to run a full pass through the OpenWorm software stack.") print( "This depends on several other repositories being loaded to work and presumes it is running in a preloaded Docker instance." ) @@ -25,54 +23,9 @@ print("Eventually all the steps will be filled in.") print("") -print("****************************") -print("Step 1: Rebuild c302 from the latest owmeta") -print("****************************") -print("Not yet implemented. See https://github.com/openworm/c302/issues/10") - - -print("****************************") -print("Step 2: Execute unit tests via the c302 simulation framework") -print("****************************") -""" -from runAndPlot import run_c302 -orig_display_var = None -if os.environ.has_key('DISPLAY'): - orig_display_var = os.environ['DISPLAY'] - del os.environ['DISPLAY'] # https://www.neuron.yale.edu/phpBB/viewtopic.php?f=6&t=1603 - -run_c302(DEFAULTS['reference'], - DEFAULTS['c302params'], - '', - DEFAULTS['duration'], - DEFAULTS['dt'], - 'jNeuroML_NEURON', - data_reader=DEFAULTS['datareader'], - save=True, - show_plot_already=False, - target_directory=os.path.join(os.environ['C302_HOME'], 'examples'), - save_fig_to='tmp_images') -prev_dir = os.getcwd() -os.chdir(DEFAULTS['outDir']) -try: - os.mkdir('c302_out') -except OSError as e: - if e.errno != errno.EEXIST: - raise -src_files = os.listdir(os.path.join(os.environ['C302_HOME'], 'examples', 'tmp_images')) -for file_name in src_files: - full_file_name = os.path.join(os.environ['C302_HOME'], 'examples', 'tmp_images', file_name) - print("COPY %s" % full_file_name) - if (os.path.isfile(full_file_name)): - shutil.copy2(full_file_name, 'c302_out') -shutil.rmtree(os.path.join(os.environ['C302_HOME'], 'examples', 'tmp_images')) -os.chdir(prev_dir) -if orig_display_var: - os.environ['DISPLAY'] = orig_display_var -""" print("****************************") -print("Step 3: Run c302 + Sibernetic in the same loop.") +print(" Step 1: Run c302 + Sibernetic in the same loop.") print("****************************") OW_OUT_DIR = os.environ["OW_OUT_DIR"] @@ -112,8 +65,6 @@ def execute_with_realtime_output(command, directory, env=None): except Exception: print("Unexpected error: %s" % sys.exc_info()[0]) -OW_OUT_DIR = os.environ["OW_OUT_DIR"] - try: if os.access(OW_OUT_DIR, os.W_OK) is not True: @@ -130,20 +81,26 @@ def execute_with_realtime_output(command, directory, env=None): if "DURATION" in os.environ: sim_duration = float(os.environ["DURATION"]) -noc302 = False + +configuration = "worm_crawl_half_resolution" +if "CONFIGURATION" in os.environ: + configuration = os.environ["CONFIGURATION"] + if "NOC302" in os.environ: - noc302 = bool(os.environ["NOC302"]) + noc302 = os.environ["NOC302"] == "1" +else: + noc302 = "worm" not in configuration -DEFAULTS = { +PARAMETERS = { "duration": sim_duration, "dt": 0.005, "dtNrn": 0.05, "logstep": 100, - "reference": "FW", + "reference": "FW", # "TargetMuscle", "c302params": "C2", "verbose": False, "device": "CPU", - "configuration": "worm_crawl_half_resolution", + "configuration": configuration, "noc302": noc302, "datareader": "UpdatedSpreadsheetDataReader2", "outDir": OW_OUT_DIR, @@ -161,6 +118,10 @@ def execute_with_realtime_output(command, directory, env=None): ) # TODO: terminate xvfb after recording try: + print("Starting Sibernetic simulation with parameters:") + for p in PARAMETERS: + print(" %s: %s" % (p, PARAMETERS[p])) + command = """python3 sibernetic_c302.py -duration %s -dt %s @@ -172,23 +133,24 @@ def execute_with_realtime_output(command, directory, env=None): -c302params %s -datareader %s -outDir %s""" % ( - DEFAULTS["duration"], - DEFAULTS["dt"], - DEFAULTS["dtNrn"], - DEFAULTS["logstep"], - DEFAULTS["device"], - DEFAULTS["configuration"], - DEFAULTS["reference"], - DEFAULTS["c302params"], - DEFAULTS["datareader"], + PARAMETERS["duration"], + PARAMETERS["dt"], + PARAMETERS["dtNrn"], + PARAMETERS["logstep"], + PARAMETERS["device"], + PARAMETERS["configuration"], + PARAMETERS["reference"], + PARAMETERS["c302params"], + PARAMETERS["datareader"], "simulations", ) - # DEFAULTS['outDir']) + # PARAMETERS['outDir']) if noc302: command += " -noc302" execute_with_realtime_output(command, os.environ["SIBERNETIC_HOME"], env=my_env) + except KeyboardInterrupt: pass @@ -198,7 +160,7 @@ def execute_with_realtime_output(command, directory, env=None): for dirpath, dirnames, filenames in os.walk(sibernetic_sim_dir): for directory in dirnames: if directory.startswith( - "%s_%s" % (DEFAULTS["c302params"], DEFAULTS["reference"]) + "%s_%s" % (PARAMETERS["c302params"], PARAMETERS["reference"]) ): all_subdirs.append(os.path.join(dirpath, directory)) if directory.startswith("Sibernetic"): @@ -277,7 +239,7 @@ def execute_with_realtime_output(command, directory, env=None): ) command = "./Release/Sibernetic -f %s -l_from lpath=%s" % ( - DEFAULTS["configuration"], + PARAMETERS["configuration"], latest_subdir, ) execute_with_realtime_output(command, os.environ["SIBERNETIC_HOME"], env=my_env) @@ -360,7 +322,7 @@ def execute_with_realtime_output(command, directory, env=None): print("****************************") -print("Step 4: Run movement analysis") +print(" Step 2: Run movement analysis") print("****************************") print("Not yet implemented.") print( @@ -370,6 +332,6 @@ def execute_with_realtime_output(command, directory, env=None): print("****************************") -print("Step 5: Report on movement analysis fit to real worm videos") +print(" Step 3: Report on movement analysis fit to real worm videos") print("****************************") print("Not yet implemented.") diff --git a/run.sh b/run.sh index 2264888..a6623bd 100755 --- a/run.sh +++ b/run.sh @@ -2,15 +2,18 @@ version=$(&2 + n) C302_PART="-e NOC302=1" + ;; + *) echo "Usage: $0 [-d duration_in_ms] [-c configuration] [-n]" >&2 exit 1 ;; esac @@ -36,6 +39,15 @@ else #Configuration is set, use it. CONFIGURATION_PART="-e CONFIGURATION=$configuration" fi +echo "Running OpenWorm Docker container. Additional options: $DURATION_PART $CONFIGURATION_PART $C302_PART" +# Check and get list of all running containers +output=$(docker ps -a) + +# Check if the openworm container is already running +if echo "$output" | grep -q "openworm_$version"; then + echo -e "\n**************\n Docker container openworm_$version is already running.\n Please stop and remove it before running a new one (run: ./stop.sh).\n**************" + exit 1 +fi echo "Running Docker container for OpenWorm v${version}" @@ -46,6 +58,7 @@ docker run -d \ -e DISPLAY=$DISPLAY \ $CONFIGURATION_PART \ $DURATION_PART \ +$C302_PART \ -e OW_OUT_DIR=$OW_OUT_DIR \ -v /tmp/.X11-unix:/tmp/.X11-unix:rw \ --privileged \ From 7170d2603360d3b9f62d2f86b6b16f4a081aedfa Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Thu, 4 Dec 2025 17:44:59 +0000 Subject: [PATCH 06/18] Improved run scripts --- run-quick.sh | 3 ++- run.sh | 1 + test.sh | 7 ++++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/run-quick.sh b/run-quick.sh index 8d78b6f..8e98c89 100755 --- a/run-quick.sh +++ b/run-quick.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -ex #from: https://unix.stackexchange.com/a/129401 while getopts ":d:p:" opt; do @@ -33,6 +34,6 @@ $DURATION_PART \ --privileged \ -v $HOST_OUT_DIR:$OW_OUT_DIR:rw \ openworm/openworm:$version \ -bash -c "DISPLAY=:44 python master_openworm.py" +bash -c "DISPLAY=:44 python3 master_openworm.py" docker logs -f openworm_$version diff --git a/run.sh b/run.sh index a6623bd..0679240 100755 --- a/run.sh +++ b/run.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -ex version=$( Date: Thu, 4 Dec 2025 17:57:58 +0000 Subject: [PATCH 07/18] Install x11-xserver-utils --- .github/workflows/docker-image-intel.yml | 1 + .github/workflows/docker-image-quickrun.yml | 1 + .github/workflows/docker-image.yml | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-image-intel.yml b/.github/workflows/docker-image-intel.yml index ff0975f..db33655 100644 --- a/.github/workflows/docker-image-intel.yml +++ b/.github/workflows/docker-image-intel.yml @@ -28,6 +28,7 @@ jobs: - name: Run the Docker image run: | + sudo apt-get update && sudo apt-get install -y x11-xserver-utils # for xhost ./run.sh - name: Info on generated files diff --git a/.github/workflows/docker-image-quickrun.yml b/.github/workflows/docker-image-quickrun.yml index 4c07b99..97f8553 100644 --- a/.github/workflows/docker-image-quickrun.yml +++ b/.github/workflows/docker-image-quickrun.yml @@ -25,6 +25,7 @@ jobs: docker images - name: Run the Docker image run: | + sudo apt-get update && sudo apt-get install -y x11-xserver-utils # for xhost ./run-quick.sh - name: Info on generated files run: | diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 2f9b2b3..b886ba0 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -28,6 +28,7 @@ jobs: - name: Run the Docker image with the default falling cube demo from sibernetic run: | + sudo apt-get update && sudo apt-get install -y x11-xserver-utils # for xhost ./run.sh -d 20 -c demo1 - name: Run the Docker image with a worm simulation - no c302 @@ -38,7 +39,7 @@ jobs: - name: Run the Docker image with a worm simulation run: | ./stop.sh - ./run.sh -d 10 + ./run.sh -d 150 # temp longer simulation for testing - name: Info on generated files run: | From de352fb045d1c061fe2a660f2ef1b1e8b3040932 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Thu, 4 Dec 2025 18:16:56 +0000 Subject: [PATCH 08/18] Better run scripts --- run-quick.sh | 2 +- run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/run-quick.sh b/run-quick.sh index 8e98c89..427fa6e 100755 --- a/run-quick.sh +++ b/run-quick.sh @@ -17,7 +17,7 @@ OW_OUT_DIR=/home/ow/shared HOST_OUT_DIR=$PWD version=$( Date: Thu, 4 Dec 2025 22:55:48 +0000 Subject: [PATCH 09/18] Pass on error code --- master_openworm.py | 2 +- run.sh | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/master_openworm.py b/master_openworm.py index 5c12e5f..4a7d120 100644 --- a/master_openworm.py +++ b/master_openworm.py @@ -54,7 +54,7 @@ def execute_with_realtime_output(command, directory, env=None): print(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n") if p.returncode != 0: - print("Exiting as the last command failed") + print("Exiting master Python script as the last command failed") exit(p.returncode) diff --git a/run.sh b/run.sh index 72db04a..929a837 100755 --- a/run.sh +++ b/run.sh @@ -67,4 +67,18 @@ $C302_PART \ openworm/openworm:$version \ bash -c "DISPLAY=:44 python3 master_openworm.py" +echo "Finished running docker deamon. Attaching to logs now:" + docker logs -f openworm_$version + +echo "Finished running Docker container" + +last_dir=$(ls -td output/*/ | head -n 1) +echo Last simulation directory: $last_dir + +if grep -q "ompleted successfully" $last_dir/report.json; then + echo "Simulation has completed successfully." +else + echo "Simulation has exited with an error." + exit -1 +fi \ No newline at end of file From de7aba8b2412b7386bc68c761414132a51540f45 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Mon, 8 Dec 2025 10:32:13 +0000 Subject: [PATCH 10/18] Better checks on failure --- .gitignore | 1 + master_openworm.py | 275 ++++++++++++++++++++++++--------------------- run.sh | 6 +- 3 files changed, 148 insertions(+), 134 deletions(-) diff --git a/.gitignore b/.gitignore index b8e0b5e..5c43697 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ output/C2* /output/alone* /output/cube* /output/master* +/output/deep_swim* diff --git a/master_openworm.py b/master_openworm.py index 4a7d120..f7e79db 100644 --- a/master_openworm.py +++ b/master_openworm.py @@ -31,7 +31,7 @@ OW_OUT_DIR = os.environ["OW_OUT_DIR"] -def execute_with_realtime_output(command, directory, env=None): +def execute_with_realtime_output(command, directory, env=None, exit_on_failure=True): p = None try: print(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>") @@ -55,7 +55,12 @@ def execute_with_realtime_output(command, directory, env=None): if p.returncode != 0: print("Exiting master Python script as the last command failed") - exit(p.returncode) + if exit_on_failure: + sys.exit(p.returncode) + else: + return False + + return True sys.path.append(os.environ["C302_HOME"]) @@ -96,7 +101,7 @@ def execute_with_realtime_output(command, directory, env=None): "dt": 0.005, "dtNrn": 0.05, "logstep": 100, - "reference": "FW", # "TargetMuscle", + "reference": "FW", # "TargetMuscle", "c302params": "C2", "verbose": False, "device": "CPU", @@ -149,7 +154,9 @@ def execute_with_realtime_output(command, directory, env=None): if noc302: command += " -noc302" - execute_with_realtime_output(command, os.environ["SIBERNETIC_HOME"], env=my_env) + success = execute_with_realtime_output( + command, os.environ["SIBERNETIC_HOME"], env=my_env, exit_on_failure=False + ) except KeyboardInterrupt: pass @@ -182,116 +189,110 @@ def execute_with_realtime_output(command, directory, env=None): new_sim_out = "%s/output/%s" % (OW_OUT_DIR, os.path.split(latest_subdir)[-1]) try: os.mkdir(new_sim_out) + print("Created new output directory: %s" % new_sim_out) except OSError as e: if e.errno != errno.EEXIST: raise +if success: + # Copy PNGs, created during the Sibernetic simulation, in a separate child-directory to find them more easily + figures = glob.glob("%s/*.png" % latest_subdir) + for figure in figures: + print("Moving %s to %s" % (figure, new_sim_out)) + shutil.move(figure, new_sim_out) + + # Copy reports etc. + reports = glob.glob("%s/report*" % latest_subdir) + for report in reports: + print("Moving %s to %s" % (report, new_sim_out)) + shutil.move(report, new_sim_out) + + # Copy position files etc. + txt_files = glob.glob("%s/*.txt" % latest_subdir) + for txt_file in txt_files: + print("Moving %s to %s" % (txt_file, new_sim_out)) + shutil.move(txt_file, new_sim_out) + dat_files = glob.glob("%s/*.dat" % latest_subdir) + for dat_file in dat_files: + print("Moving %s to %s" % (dat_file, new_sim_out)) + shutil.move(dat_file, new_sim_out) + + # Copy WCON file(s) + wcons = glob.glob("%s/*.wcon" % latest_subdir) + for wcon in wcons: + print("Moving %s to %s" % (wcon, new_sim_out)) + shutil.move(wcon, new_sim_out) + + time.sleep(2) + + # Rerun and record simulation + execute_with_realtime_output( + "ls -alt /tmp/.X11-unix", os.environ["SIBERNETIC_HOME"], env=my_env + ) + os.system("export DISPLAY=%s" % DISPLAY) + execute_with_realtime_output( + "ls -alt /tmp/.X11-unix", os.environ["SIBERNETIC_HOME"], env=my_env + ) + sibernetic_movie_name = "%s.mp4" % os.path.split(latest_subdir)[-1] + command = ( + 'tmux new-session -d -P -s SiberneticRecording "DISPLAY=%s ffmpeg -r 30 -f x11grab -draw_mouse 0 -s 1920x1080 -i %s -filter:v "crop=1200:800:100:100" -cpu-used 0 -b:v 384k -qmin 10 -qmax 42 -maxrate 384k -bufsize 1000k -an %s/%s"' + % (DISPLAY, DISPLAY, new_sim_out, sibernetic_movie_name) + ) + execute_with_realtime_output(command, os.environ["SIBERNETIC_HOME"], env=my_env) -# Copy PNGs, created during the Sibernetic simulation, in a separate child-directory to find them more easily -figures = glob.glob("%s/*.png" % latest_subdir) -for figure in figures: - print("Moving %s to %s" % (figure, new_sim_out)) - shutil.move(figure, new_sim_out) - -# Copy reports etc. -reports = glob.glob("%s/report*" % latest_subdir) -for report in reports: - print("Moving %s to %s" % (report, new_sim_out)) - shutil.move(report, new_sim_out) - -# Copy position files etc. -txt_files = glob.glob("%s/*.txt" % latest_subdir) -for txt_file in txt_files: - print("Moving %s to %s" % (txt_file, new_sim_out)) - shutil.move(txt_file, new_sim_out) -dat_files = glob.glob("%s/*.dat" % latest_subdir) -for dat_file in dat_files: - print("Moving %s to %s" % (dat_file, new_sim_out)) - shutil.move(dat_file, new_sim_out) - -# Copy WCON file(s) -wcons = glob.glob("%s/*.wcon" % latest_subdir) -for wcon in wcons: - print("Moving %s to %s" % (wcon, new_sim_out)) - shutil.move(wcon, new_sim_out) - -time.sleep(2) - -# Rerun and record simulation -execute_with_realtime_output( - "ls -alt /tmp/.X11-unix", os.environ["SIBERNETIC_HOME"], env=my_env -) -os.system("export DISPLAY=%s" % DISPLAY) -execute_with_realtime_output( - "ls -alt /tmp/.X11-unix", os.environ["SIBERNETIC_HOME"], env=my_env -) -sibernetic_movie_name = "%s.mp4" % os.path.split(latest_subdir)[-1] -command = ( - 'tmux new-session -d -P -s SiberneticRecording "DISPLAY=%s ffmpeg -r 30 -f x11grab -draw_mouse 0 -s 1920x1080 -i %s -filter:v "crop=1200:800:100:100" -cpu-used 0 -b:v 384k -qmin 10 -qmax 42 -maxrate 384k -bufsize 1000k -an %s/%s"' - % (DISPLAY, DISPLAY, new_sim_out, sibernetic_movie_name) -) -execute_with_realtime_output(command, os.environ["SIBERNETIC_HOME"], env=my_env) - -time.sleep(3) + time.sleep(3) -execute_with_realtime_output( - "tmux list-sessions", os.environ["SIBERNETIC_HOME"], env=my_env -) + execute_with_realtime_output( + "tmux list-sessions", os.environ["SIBERNETIC_HOME"], env=my_env + ) -command = "./Release/Sibernetic -f %s -l_from lpath=%s" % ( - PARAMETERS["configuration"], - latest_subdir, -) -execute_with_realtime_output(command, os.environ["SIBERNETIC_HOME"], env=my_env) + command = "./Release/Sibernetic -f %s -l_from lpath=%s" % ( + PARAMETERS["configuration"], + latest_subdir, + ) + execute_with_realtime_output(command, os.environ["SIBERNETIC_HOME"], env=my_env) -execute_with_realtime_output( - "tmux send-keys -t SiberneticRecording q", os.environ["SIBERNETIC_HOME"], env=my_env -) -execute_with_realtime_output( - 'tmux send-keys -t SiberneticRecording "exit" C-m', - os.environ["SIBERNETIC_HOME"], - env=my_env, -) + execute_with_realtime_output( + "tmux send-keys -t SiberneticRecording q", + os.environ["SIBERNETIC_HOME"], + env=my_env, + ) + execute_with_realtime_output( + 'tmux send-keys -t SiberneticRecording "exit" C-m', + os.environ["SIBERNETIC_HOME"], + env=my_env, + ) -time.sleep(3) + time.sleep(3) -execute_with_realtime_output( - "ls -alt %s" % latest_subdir, os.environ["SIBERNETIC_HOME"], env=my_env -) + execute_with_realtime_output( + "ls -alt %s" % latest_subdir, os.environ["SIBERNETIC_HOME"], env=my_env + ) -# Remove black frames at the beginning of the recorded video -command = ( - "ffmpeg -i %s/%s -vf blackdetect=d=0:pic_th=0.70:pix_th=0.10 -an -f null - 2>&1 | grep blackdetect" - % (new_sim_out, sibernetic_movie_name) -) -outstr = str(check_output(command, shell=True).decode("utf-8")) -outstr = outstr.split("\n") - -black_start = 0.0 -black_dur = None - - -out = outstr[0] - -black_start_pos = out.find("black_start:") -black_end_pos = out.find("black_end:") -black_dur_pos = out.find("black_duration:") -if black_start_pos != -1: - black_start = float(out[black_start_pos + len("black_start:") : black_end_pos]) - black_dur = float(out[black_dur_pos + len("black_duration:") :]) - -if black_start == 0.0 and black_dur: - black_dur = math.ceil(black_dur) - command = "ffmpeg -ss 00:00:0%s -i %s/%s -c copy -avoid_negative_ts 1 %s/cut_%s" % ( - black_dur, - new_sim_out, - sibernetic_movie_name, - new_sim_out, - sibernetic_movie_name, + # Remove black frames at the beginning of the recorded video + command = ( + "ffmpeg -i %s/%s -vf blackdetect=d=0:pic_th=0.70:pix_th=0.10 -an -f null - 2>&1 | grep blackdetect" + % (new_sim_out, sibernetic_movie_name) ) - if black_dur > 9: + outstr = str(check_output(command, shell=True).decode("utf-8")) + outstr = outstr.split("\n") + + black_start = 0.0 + black_dur = None + + out = outstr[0] + + black_start_pos = out.find("black_start:") + black_end_pos = out.find("black_end:") + black_dur_pos = out.find("black_duration:") + if black_start_pos != -1: + black_start = float(out[black_start_pos + len("black_start:") : black_end_pos]) + black_dur = float(out[black_dur_pos + len("black_duration:") :]) + + if black_start == 0.0 and black_dur: + black_dur = math.ceil(black_dur) command = ( - "ffmpeg -ss 00:00:%s -i %s/%s -c copy -avoid_negative_ts 1 %s/cut_%s" + "ffmpeg -ss 00:00:0%s -i %s/%s -c copy -avoid_negative_ts 1 %s/cut_%s" % ( black_dur, new_sim_out, @@ -300,38 +301,50 @@ def execute_with_realtime_output(command, directory, env=None): sibernetic_movie_name, ) ) - os.system(command) - -# SPEED-UP -try: - os.mkdir("tmp") -except OSError as e: - if e.errno != errno.EEXIST: - raise - -os.system( - 'ffmpeg -ss 1 -i %s/cut_%s -vf "select=gt(scene\,0.1)" -vsync vfr -vf fps=fps=1/1 %s' - % (new_sim_out, sibernetic_movie_name, "tmp/out%06d.jpg") -) -os.system( - "ffmpeg -r 100 -i %s -r 100 -vb 60M %s/speeded_%s" - % ("tmp/out%06d.jpg", new_sim_out, sibernetic_movie_name) -) + if black_dur > 9: + command = ( + "ffmpeg -ss 00:00:%s -i %s/%s -c copy -avoid_negative_ts 1 %s/cut_%s" + % ( + black_dur, + new_sim_out, + sibernetic_movie_name, + new_sim_out, + sibernetic_movie_name, + ) + ) + os.system(command) -os.system("rm -r tmp/*") + # SPEED-UP + try: + os.mkdir("tmp") + except OSError as e: + if e.errno != errno.EEXIST: + raise + + os.system( + 'ffmpeg -ss 1 -i %s/cut_%s -vf "select=gt(scene\,0.1)" -vsync vfr -vf fps=fps=1/1 %s' + % (new_sim_out, sibernetic_movie_name, "tmp/out%06d.jpg") + ) + os.system( + "ffmpeg -r 100 -i %s -r 100 -vb 60M %s/speeded_%s" + % ("tmp/out%06d.jpg", new_sim_out, sibernetic_movie_name) + ) + os.system("rm -r tmp/*") -print("****************************") -print(" Step 2: Run movement analysis") -print("****************************") -print("Not yet implemented.") -print( - "Note however the following WCON files have been generated into %s during the simulation: %s" - % (new_sim_out, [w.split("/")[-1] for w in wcons]) -) + print("****************************") + print(" Step 2: Run movement analysis") + print("****************************") + print("Not yet implemented.") + print( + "Note however the following WCON files have been generated into %s during the simulation: %s" + % (new_sim_out, [w.split("/")[-1] for w in wcons]) + ) + print("****************************") + print(" Step 3: Report on movement analysis fit to real worm videos") + print("****************************") + print("Not yet implemented.") -print("****************************") -print(" Step 3: Report on movement analysis fit to real worm videos") -print("****************************") -print("Not yet implemented.") +else: + print("Simulation failed, skipping other steps.") diff --git a/run.sh b/run.sh index 929a837..5beebe8 100755 --- a/run.sh +++ b/run.sh @@ -67,14 +67,14 @@ $C302_PART \ openworm/openworm:$version \ bash -c "DISPLAY=:44 python3 master_openworm.py" -echo "Finished running docker deamon. Attaching to logs now:" +echo "Set running running Docker container with Sibernetic in detached mode. Attaching to logs now..." docker logs -f openworm_$version -echo "Finished running Docker container" +echo "Finished running the Docker container" last_dir=$(ls -td output/*/ | head -n 1) -echo Last simulation directory: $last_dir +echo Last created simulation directory: $last_dir if grep -q "ompleted successfully" $last_dir/report.json; then echo "Simulation has completed successfully." From 01d3d0b07242357d7882e91d45e2dac8e3ff1fbb Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Mon, 8 Dec 2025 13:10:23 +0000 Subject: [PATCH 11/18] Update intel docker --- Dockerfile_intel | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/Dockerfile_intel b/Dockerfile_intel index e8c5291..fe9636c 100644 --- a/Dockerfile_intel +++ b/Dockerfile_intel @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 +FROM ubuntu:24.04 LABEL maintainer="David Lung (lungdm@gmail.com); Padraig Gleeson (p.gleeson@gmail.com)" @@ -18,42 +18,40 @@ RUN mkdir -p /etc/sudoers.d && \ chmod 0440 /etc/sudoers.d/$USER && \ chown ${uid}:${gid} -R /home/$USER -ENV DEBIAN_FRONTEND noninteractive # TODO: change +ENV DEBIAN_FRONTEND=noninteractive ################################################################################ ######## Update/install essential libraries -RUN apt-get update && apt-get install -y --no-install-recommends apt-utils \ +RUN apt-get update +RUN apt-get install -y --no-install-recommends apt-utils \ wget nano htop build-essential make git automake autoconf \ - g++ rpm libtool libncurses5-dev zlib1g-dev bison flex lsb-core \ + g++ rpm libtool libncurses5-dev zlib1g-dev bison flex \ sudo xorg openbox x11-xserver-utils \ - libxext-dev libncurses-dev python3-dev mercurial \ - freeglut3-dev libglu1-mesa-dev libglew-dev python3-dev python3-pip python3-lxml python3-scipy python3-tk \ + libxext-dev libncurses-dev mercurial \ + freeglut3-dev libglu1-mesa-dev libglew-dev python3-dev python3-pip \ kmod dkms linux-source linux-headers-generic \ maven openjdk-8-jdk \ - python3-setuptools python3-yaml libnuma1 \ + libnuma1 \ openmpi-bin libopenmpi-dev \ - libgl1-mesa-glx libgl1-mesa-dri libfreetype6-dev \ - libxft-dev python3-matplotlib unzip ffmpeg xvfb tmux + libgl1 libglx-mesa0 libgl1-mesa-dri libfreetype6-dev \ + libxft-dev unzip ffmpeg xvfb tmux #RUN sudo pip install --upgrade pip RUN sudo usermod -a -G video $USER #USER $USER -ENV HOME /home/$USER +ENV HOME=/home/$USER WORKDIR $HOME -#### TODO: check that this is the best way to switch to py3... -RUN sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10 -RUN sudo update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 10 ################################################################################ ######## Install NEURON simulator -RUN pip3 install neuron==8.2.6 +RUN sudo pip install neuron==8.2.6 --break-system-packages ################################################################################ @@ -61,13 +59,11 @@ RUN pip3 install neuron==8.2.6 RUN git clone https://github.com/openworm/c302.git && \ cd c302 && \ - git checkout ow-0.9.5 && \ - sudo pip install . + git checkout ow-0.9.7 && \ + sudo pip install . --break-system-packages # Note: pyNeuroML installed with the above library -RUN pip3 install owmeta-core==0.13.5 -RUN owm bundle remote --user add ow 'https://raw.githubusercontent.com/openworm/owmeta-bundles/master/index.json' ################################################################################ @@ -75,7 +71,13 @@ RUN owm bundle remote --user add ow 'https://raw.githubusercontent.com/openworm/ RUN git clone https://github.com/openworm/sibernetic.git && \ cd sibernetic && \ - git checkout ow-0.9.5 # fixed to a specific branch + git checkout ow-0.9.7 # fixed to a specific branch + + +################################################################################ +######## Install extra Python dependencies + +RUN sudo pip install ruff --break-system-packages ################################################################################ @@ -83,7 +85,9 @@ RUN git clone https://github.com/openworm/sibernetic.git && \ ENV C302_HOME=$HOME/c302/c302 ENV SIBERNETIC_HOME=$HOME/sibernetic -ENV PYTHONPATH=$PYTHONPATH:$HOME/c302:$SIBERNETIC_HOME +ENV PYTHONPATH=$HOME/c302:$SIBERNETIC_HOME + +ENV NEURON_MODULE_OPTIONS=-nogui ################################################################################ @@ -124,7 +128,7 @@ RUN cd sibernetic && \ COPY ./master_openworm.py $HOME/master_openworm.py RUN sudo chown $USER:$USER $HOME/master_openworm.py -RUN echo '\n\nalias cd..="cd .."\nalias h=history\nalias ll="ls -alt"' >> ~/.bashrc +RUN printf '\n\nalias cd..="cd .."\nalias h=history\nalias ll="ls -alth"\n' >> ~/.bashrc RUN pip list From cdd360ecc5d73f2387b9cba62f6f5265d89fd849 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Mon, 8 Dec 2025 13:10:34 +0000 Subject: [PATCH 12/18] Run shorter sim for testing --- .github/workflows/docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index b886ba0..45933b6 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -39,7 +39,7 @@ jobs: - name: Run the Docker image with a worm simulation run: | ./stop.sh - ./run.sh -d 150 # temp longer simulation for testing + ./run.sh -d 15 # temp longer simulation for testing - name: Info on generated files run: | From 2c63d33dd6b951741fc3de22ab4d5d39a88127d5 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Mon, 8 Dec 2025 13:10:47 +0000 Subject: [PATCH 13/18] Improved intel build --- build_intel.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/build_intel.sh b/build_intel.sh index 8553e74..b8af92e 100755 --- a/build_intel.sh +++ b/build_intel.sh @@ -3,4 +3,14 @@ # A version of the docker container using the Intel libraries for OpenCL version=$( Date: Mon, 8 Dec 2025 18:19:26 +0000 Subject: [PATCH 14/18] Skipping actual run of Intel Docker image, due to segfaults observed on GitHub runners --- .github/workflows/docker-image-intel.yml | 12 +++++------- .github/workflows/docker-image-quickrun.yml | 4 ++-- Dockerfile_intel | 5 +++++ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-image-intel.yml b/.github/workflows/docker-image-intel.yml index db33655..53c5dea 100644 --- a/.github/workflows/docker-image-intel.yml +++ b/.github/workflows/docker-image-intel.yml @@ -2,9 +2,9 @@ name: Docker Image Build - Intel drivers on: push: - branches: [ master, dev*, test* ] + branches: [ master, dev*, test*, experimental ] pull_request: - branches: [ master, dev*, test* ] + branches: [ master, dev*, test*, experimental ] jobs: @@ -28,12 +28,10 @@ jobs: - name: Run the Docker image run: | - sudo apt-get update && sudo apt-get install -y x11-xserver-utils # for xhost - ./run.sh + echo "Skipping actual run of Intel Docker image, due to segfaults observed on GitHub runners" + #sudo apt-get update && sudo apt-get install -y x11-xserver-utils # for xhost + #./run.sh - name: Info on generated files run: | ls -alth - ls -alth output - ls -alth output/* - more output/*/report.json diff --git a/.github/workflows/docker-image-quickrun.yml b/.github/workflows/docker-image-quickrun.yml index 97f8553..5ab9b37 100644 --- a/.github/workflows/docker-image-quickrun.yml +++ b/.github/workflows/docker-image-quickrun.yml @@ -2,9 +2,9 @@ name: Docker Image Quick Run on: push: - branches: [ master, dev*, test* ] + branches: [ master, dev*, test*, experimental ] pull_request: - branches: [ master, dev*, test* ] + branches: [ master, dev*, test*, experimental ] jobs: diff --git a/Dockerfile_intel b/Dockerfile_intel index fe9636c..1109ecd 100644 --- a/Dockerfile_intel +++ b/Dockerfile_intel @@ -1,5 +1,10 @@ FROM ubuntu:24.04 +################################################################################ +######## Note: main Dockerfile using AMD libraries should be used unless +######## there is a specific reason to use Intel OpenCL libraries +################################################################################ + LABEL maintainer="David Lung (lungdm@gmail.com); Padraig Gleeson (p.gleeson@gmail.com)" ARG USR=ow From e7b5252c673f28530e658c7614b1411df2525551 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Mon, 8 Dec 2025 18:48:47 +0000 Subject: [PATCH 15/18] Back to longer test --- .github/workflows/docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 45933b6..b886ba0 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -39,7 +39,7 @@ jobs: - name: Run the Docker image with a worm simulation run: | ./stop.sh - ./run.sh -d 15 # temp longer simulation for testing + ./run.sh -d 150 # temp longer simulation for testing - name: Info on generated files run: | From 39cb02d22841d7842ce0203f213e9033c9a885b7 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Tue, 9 Dec 2025 13:32:04 +0000 Subject: [PATCH 16/18] Update to use latest c302, sibernetic --- Dockerfile | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c8f4616..e75c4d2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,7 +59,7 @@ RUN sudo pip install neuron==8.2.6 --break-system-packages RUN git clone https://github.com/openworm/c302.git && \ cd c302 && \ - git checkout ow-0.9.7 && \ + git checkout ow-0.9.8 && \ sudo pip install . --break-system-packages # Note: pyNeuroML installed with the above library @@ -71,7 +71,7 @@ RUN git clone https://github.com/openworm/c302.git && \ RUN git clone https://github.com/openworm/sibernetic.git && \ cd sibernetic && \ - git checkout ow-0.9.7 # fixed to a specific branch + git checkout ow-0.9.8a # fixed to a specific branch ################################################################################ @@ -116,6 +116,12 @@ RUN cd sibernetic && \ make clean && make all && ldd ./Release/Sibernetic # Use python 3 libs +################################################################################ +######## Set up JupyterLab + +#RUN sudo pip install notebook jupyterlab --break-system-packages + + ################################################################################ ######## Copy master python script @@ -125,6 +131,9 @@ RUN sudo chown $USER:$USER $HOME/master_openworm.py RUN printf '\n\nalias cd..="cd .."\nalias h=history\nalias ll="ls -alth"\n' >> ~/.bashrc + + RUN pip list RUN echo "Built the OpenWorm Docker image!" + From beabef592a16fac6370ba44d40f871de050c83fc Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Tue, 9 Dec 2025 17:14:55 +0000 Subject: [PATCH 17/18] Test intel on ubuntu-22.04 --- .github/workflows/docker-image-intel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-image-intel.yml b/.github/workflows/docker-image-intel.yml index 53c5dea..b9b4eae 100644 --- a/.github/workflows/docker-image-intel.yml +++ b/.github/workflows/docker-image-intel.yml @@ -10,7 +10,7 @@ jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 From 5ca633be2b4320ae9d982458f2290c63cb3ef12e Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Tue, 9 Dec 2025 17:15:14 +0000 Subject: [PATCH 18/18] Quieter running of simulation by default --- master_openworm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/master_openworm.py b/master_openworm.py index f7e79db..0a07e4f 100644 --- a/master_openworm.py +++ b/master_openworm.py @@ -128,6 +128,7 @@ def execute_with_realtime_output(command, directory, env=None, exit_on_failure=T print(" %s: %s" % (p, PARAMETERS[p])) command = """python3 sibernetic_c302.py + -q -duration %s -dt %s -dtNrn %s