Merge pull request #325 from openworm/dev_0.9.2

Restructure to make versioning easier

Author: pgleeson

.github/workflows/docker-image-quickrun.yml

@@ -0,0 +1,33 @@
+name: Docker Image Test - quick
+
+on:
+  push:
+    branches: [ master, dev* ]
+  pull_request:
+    branches: [ master, dev* ]
+
+jobs:
+
+  build:
+
+    runs-on: [ ubuntu-18.04, ubuntu-20.04, ubuntu-22.04 ]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build the Docker image
+      run: |
+        chmod a+w output/
+        ls -alt
+        ./build.sh
+    - name: Info on Docker images
+      run: |
+        docker -v
+        docker images
+    - name: Run the Docker image
+      run: |
+        ./run-quick.sh
+    - name: Info on generated files
+      run: |
+        ls -alt
+        ls -alt output
+        ls -alt output/S*
+        cat output/S*/report.json

.github/workflows/docker-image.yml

@@ -16,12 +16,18 @@ jobs:
     - uses: actions/checkout@v2
     - name: Build the Docker image
       run: |
+        chmod a+w output/
+        ls -alt
         ./build.sh
+    - name: Info on Docker images
+      run: |
+        docker images
     - name: Run the Docker image
       run: |
         ./run.sh
     - name: Info on generated files
       run: |
+        ls -alt
         ls -alt output
         ls -alt output/C2*
         cat output/C2_*/report.json

.gitignore

@@ -3,3 +3,4 @@ output/C2*
 /rebuild2.sh
 /output/siminfo.py
 /output/wor*
+/output/Sibernetic_*

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012-2022 Individual Authors of code 
+Copyright (c) 2012-2022 Individual Authors of code
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

VERSION

@@ -0,0 +1 @@
+0.9.2

build.sh

@@ -1,3 +1,4 @@
 #!/bin/bash
 
-docker build "$@" -t "openworm/openworm:0.9.2" .
+version=$(<VERSION) # Read version of Dockerfile from file VERSION
+docker build "$@" -t "openworm/openworm:$version" .

build2.sh

@@ -4,4 +4,5 @@
 # Note: Python 2 is no longer officially supported and this Docker image will
 # probably stop working eventually...
 
-docker build "$@" -t "openworm/openworm:0.9.2_py2"  -f Dockerfile2 .
+version=$(<VERSION) # Read version of Dockerfile from file VERSION
+docker build "$@" -t "openworm/openworm:${version}_py2"  -f Dockerfile2 .

master_openworm.py

@@ -2,7 +2,7 @@
 import matplotlib
 matplotlib.use('Agg')
 import shutil
-from subprocess import Popen, PIPE, check_output
+from subprocess import Popen, PIPE, check_output, STDOUT
 import os
 import pwd
 import shlex
@@ -12,7 +12,7 @@
 import math
 
 print("*****************************")
-print("OpenWorm Master Script v0.9.2")
+print("OpenWorm Master Script v0.9.3")
 print("*****************************")
 print("")
 print("This script attempts to run a full pass through the OpenWorm scientific libraries.")
@@ -76,16 +76,32 @@
 def execute_with_realtime_output(command, directory, env=None):
     p = None
     try:
-        p = Popen(shlex.split(command), stdout=PIPE, bufsize=1, cwd=directory, env=env)
+
+        print('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+        print('>> Executing command: %s'%command)
+        print('>> --------------------------------------------------------------')
+        p = Popen(shlex.split(command),
+                  stdout=PIPE,
+                  stderr=STDOUT,
+                  cwd=directory,
+                  env=env)
         with p.stdout:
             for line in iter(p.stdout.readline, b''):
-                 print(line.decode('utf-8'), end='')     
+                 print('>>  %s'%line.decode('utf-8'), end='')
         p.wait() # wait for the subprocess to exit
     except KeyboardInterrupt as e:
         print("Caught CTRL+C")
         if p:
             p.kill()
         raise e
+    print('>> --------------------------------------------------------------')
+    print('>> Command exited with %i: %s'%(p.returncode,command))
+    print('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n')
+
+    if p.returncode!=0:
+        print('Exiting as the last command failed')
+        exit(p.returncode)
+
 
 
 sys.path.append(os.environ['C302_HOME'])
@@ -110,6 +126,10 @@ def execute_with_realtime_output(command, directory, env=None):
 if 'DURATION' in os.environ:
     sim_duration = float(os.environ['DURATION'])
 
+noc302 = False
+if 'NOC302' in os.environ:
+    noc302 = bool(os.environ['NOC302'])
+
 DEFAULTS = {'duration': sim_duration,
             'dt': 0.005,
             'dtNrn': 0.05,
@@ -119,13 +139,15 @@ def execute_with_realtime_output(command, directory, env=None):
             'verbose': False,
             'device': 'GPU',
             'configuration': 'worm_crawl_half_resolution',
-            'noc302': False,
+            'noc302': noc302,
             'datareader': 'UpdatedSpreadsheetDataReader2',
             'outDir': OW_OUT_DIR}
 
 my_env = os.environ.copy()
 my_env["DISPLAY"] = ":44"
 
+# Xvfb or X virtual framebuffer is a display server implementing the X11 display server protocol.
+# In contrast to other display servers, Xvfb performs all graphical operations in virtual memory without showing any screen output.
 os.system('Xvfb :44 -listen tcp -ac -screen 0 1920x1080x24+32 &') # TODO: terminate xvfb after recording
 
 try:
@@ -151,6 +173,9 @@ def execute_with_realtime_output(command, directory, env=None):
                 DEFAULTS['datareader'],
                 'simulations')
                 #DEFAULTS['outDir'])
+
+    if noc302: command += ' -noc302'
+
     execute_with_realtime_output(command, os.environ['SIBERNETIC_HOME'], env=my_env)
 except KeyboardInterrupt as e:
     pass
@@ -162,9 +187,13 @@ def execute_with_realtime_output(command, directory, env=None):
     for directory in dirnames:
         if directory.startswith('%s_%s' % (DEFAULTS['c302params'], DEFAULTS['reference'])):
             all_subdirs.append(os.path.join(dirpath, directory))
+        if directory.startswith('Sibernetic'):
+            all_subdirs.append(os.path.join(dirpath, directory))
 
 latest_subdir = max(all_subdirs, key=os.path.getmtime)
-
+print('\n========================================================================\n')
+print('Finished main simulation, data saved in: %s'%latest_subdir)
+execute_with_realtime_output('ls -alt %s'%latest_subdir, os.environ['SIBERNETIC_HOME'], env=my_env)
 
 try:
     os.mkdir('%s/output' % OW_OUT_DIR)
@@ -198,19 +227,27 @@ def execute_with_realtime_output(command, directory, env=None):
     print("Moving %s to %s"%(wcon, new_sim_out))
     shutil.move(wcon, new_sim_out)
 
+time.sleep(2)
 
 # Rerun and record simulation
 os.system('export DISPLAY=:44')
 sibernetic_movie_name = '%s.mp4' % os.path.split(latest_subdir)[-1]
-os.system('tmux new-session -d -s SiberneticRecording "DISPLAY=:44 ffmpeg -r 30 -f x11grab -draw_mouse 0 -s 1920x1080 -i :44 -filter:v "crop=1200:800:100:100" -cpu-used 0 -b:v 384k -qmin 10 -qmax 42 -maxrate 384k -bufsize 1000k -an %s/%s"' % (new_sim_out, sibernetic_movie_name))
+command = 'tmux new-session -d -s SiberneticRecording "DISPLAY=:44 ffmpeg -r 30 -f x11grab -draw_mouse 0 -s 1920x1080 -i :44 -filter:v "crop=1200:800:100:100" -cpu-used 0 -b:v 384k -qmin 10 -qmax 42 -maxrate 384k -bufsize 1000k -an %s/%s"' % (new_sim_out, sibernetic_movie_name)
+execute_with_realtime_output(command, os.environ['SIBERNETIC_HOME'], env=my_env)
+
+time.sleep(3)
+
+execute_with_realtime_output('tmux list-sessions', os.environ['SIBERNETIC_HOME'], env=my_env)
 
 command = './Release/Sibernetic -f %s -l_from lpath=%s' % (DEFAULTS['configuration'], latest_subdir)
 execute_with_realtime_output(command, os.environ['SIBERNETIC_HOME'], env=my_env)
 
-os.system('tmux send-keys -t SiberneticRecording q')
-os.system('tmux send-keys -t SiberneticRecording "exit" C-m')
+execute_with_realtime_output('tmux send-keys -t SiberneticRecording q', os.environ['SIBERNETIC_HOME'], env=my_env)
+execute_with_realtime_output('tmux send-keys -t SiberneticRecording "exit" C-m', os.environ['SIBERNETIC_HOME'], env=my_env)
 
-time.sleep(3)
+time.sleep(5)
+
+execute_with_realtime_output('ls -alt %s'%latest_subdir, os.environ['SIBERNETIC_HOME'], env=my_env)
 
 # Remove black frames at the beginning of the recorded video
 command = "ffmpeg -i %s/%s -vf blackdetect=d=0:pic_th=0.70:pix_th=0.10 -an -f null - 2>&1 | grep blackdetect" % (new_sim_out, sibernetic_movie_name)
@@ -247,7 +284,7 @@ def execute_with_realtime_output(command, directory, env=None):
 os.system('ffmpeg -ss 1 -i %s/cut_%s -vf "select=gt(scene\,0.1)" -vsync vfr -vf fps=fps=1/1 %s' % (new_sim_out, sibernetic_movie_name, 'tmp/out%06d.jpg'))
 os.system('ffmpeg -r 100 -i %s -r 100 -vb 60M %s/speeded_%s' % ('tmp/out%06d.jpg', new_sim_out, sibernetic_movie_name))
 
-os.system('sudo rm -r tmp/*')
+os.system('rm -r tmp/*')
 
 
 

rebuild.sh

@@ -1,3 +1,4 @@
 #!/bin/bash
 
-docker build "$@" -t "openworm/openworm:0.9.2"  --no-cache .
+version=$(<VERSION) # Read version of Dockerfile from file VERSION
+docker build "$@" -t "openworm/openworm:$version"  --no-cache .

run-quick.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+#from: https://unix.stackexchange.com/a/129401
+while getopts ":d:p:" opt; do
+  case $opt in
+    d) duration="$OPTARG"
+    ;;
+    p) p_out="$OPTARG"
+    ;;
+    \?) echo "Invalid option -$OPTARG" >&2
+    ;;
+  esac
+done
+
+OW_OUT_DIR=/home/ow/shared
+HOST_OUT_DIR=$PWD
+version=$(<VERSION) # Read version of Dockerfile from file VERSION
+
+xhost +
+
+
+NOC302_PART="-e NOC302=1"
+DURATION_PART="-e DURATION=3"
+
+docker run -d \
+--name openworm_$version \
+--device=/dev/dri:/dev/dri \
+-e DISPLAY=$DISPLAY \
+$NOC302_PART \
+$DURATION_PART \
+-e OW_OUT_DIR=$OW_OUT_DIR \
+-v /tmp/.X11-unix:/tmp/.X11-unix:rw \
+--privileged \
+-v $HOST_OUT_DIR:$OW_OUT_DIR:rw \
+openworm/openworm:$version \
+bash -c "DISPLAY=:44 python master_openworm.py"
+
+docker logs -f openworm_$version