Integration

aurellem

Written by:

Robert McIntyre

1 Integration

This is the ultimate test which features all of the senses that I've made so far. The blender file for the creature serves as an example of a fully equipped creature in terms of senses. You can find it here.


YouTube

Simulated Senses in a Virtual Environment

2 Generating the Video

(ns cortex.test.integration
  "let's play!"
  {:author "Robert McIntyre"}
  (:use (cortex world util import body sense
                hearing touch vision proprioception movement))
  (:import (com.jme3.math ColorRGBA Vector3f))
  (:import java.io.File)
  (:import com.jme3.audio.AudioNode)
  (:import com.aurellem.capture.RatchetTimer))

(dorun (cortex.import/mega-import-jme3))
;;(rlm.rlm-commands/help)

(def hand "Models/test-creature/hand.blend")

(def output-base (File. "/home/r/proj/cortex/render/hand"))

For this demonstration I have to manually drive the muscles of the hand. I do this by creating a little mini-language to describe simulated muscle contraction.

(defn motor-control-program
  "Create a function which will execute the motor script"
  [muscle-positions
   script]
  (let [current-frame (atom -1)
        keyed-script (group-by first script)
        current-forces (atom {}) ]
    (fn [effectors]
      (let [indexed-effectors (vec effectors)]
        (dorun 
         (for [[_ part force] (keyed-script (swap! current-frame inc))]
           (swap! current-forces (fn [m] (assoc m part force)))))
        (doall (map (fn [effector power]
                      (effector (int power)))
                    effectors
                    (map #(@current-forces % 0) muscle-positions)))))))

(def muscle-positions
  [:pointer-2-e
   :pointer-2-f
   :thumb-1
   :thumb-1
   :pointer-1-e
   :pointer-1-f
   :thumb-2-e
   :thumb-2-f
   :middle-1-e
   :middle-1-f
   :pointer-3-f
   :pointer-3-e
   :middle-2-e
   :middle-2-f
   :middle-3-f
   :middle-3-e
   :pinky-2-e
   :pinky-2-f
   :pinky-3-f
   :pinky-3-e
   :ring-3-e
   :ring-3-f
   :ring-2-f
   :ring-2-e
   :ring-1-e
   :ring-1-f
   :thumb-1-e
   :thumb-1-f
   :pinky-1-f
   :pinky-1-e])

(def full 9001)


;; Choreography:

;; Let the hand fall palm-up

;; it curls its phalanges, starting with the pinky.

;; it lets its phalanges fall back down.

;; block falls down onto the hand, accompanied by a sound. The block
;; can be seen by the hand's eye.

;; hand FORCEFULLY catapults the  block so that it hits the camera.


;; the syntax here is [keyframe body-part force]
(def move-fingers
  [[300 :pinky-3-f 50]
   [320 :pinky-2-f 80]
   [340 :pinky-1-f 100]

   [310 :ring-3-f 100]
   [330 :ring-2-f 120]
   [350 :ring-1-f 140]

   [330 :middle-3-f 120]
   [340 :middle-2-f 120]
   [360 :middle-1-f 30]

   [350 :pointer-3-f 120]
   [360 :pointer-2-f 120]
   [380 :pointer-1-f 30]

   [800 :pinky-3-f 0]
   [800 :pinky-2-f 0]
   [800 :pinky-1-f  0]

   [800 :ring-3-f  0]
   [800 :ring-2-f  0]
   [800 :ring-1-f  0]

   [800 :middle-3-f  0]
   [800 :middle-2-f  0]
   [800 :middle-1-f 0]

   [800 :pointer-3-f  0]
   [800 :pointer-2-f  0]
   [800 :pointer-1-f 0]

   
   [800 :pinky-3-e 50]
   [800 :pinky-2-e 80]
   [800 :pinky-1-e 100]

   [800 :ring-3-e 100]
   [800 :ring-2-e 120]
   [800 :ring-1-e 140]

   [800 :middle-3-e 120]
   [800 :middle-2-e 120]
   [800 :middle-1-e 30]

   [800 :pointer-3-e 120]
   [800 :pointer-2-e 120]
   [800 :pointer-1-e 30]

   [870 :pinky-3-e 0]
   [870 :pinky-2-e 0]
   [870 :pinky-1-e  0]

   [870 :ring-3-e  0]
   [870 :ring-2-e  0]
   [870 :ring-1-e  0]

   [870 :middle-3-e  0]
   [870 :middle-2-e  0]
   [870 :middle-1-e 0]

   [870 :pointer-3-e  0]
   [870 :pointer-2-e  0]
   [870 :pointer-1-e 0]

   [1500 :pointer-1-f full]
   [1500 :pointer-2-f full]
   [1500 :pointer-3-f full]
   
   [1500 :middle-1-f full]
   [1500 :middle-2-f full]
   [1500 :middle-3-f full]
   
   [1510 :pointer-1-f 0]
   [1510 :pointer-2-f 0]
   [1510 :pointer-3-f 0]
   
   [1510 :middle-1-f 0]
   [1510 :middle-2-f 0]
   [1510 :middle-3-f 0]
   ])

(defn gen-summon-ball [debug?]
  (let [wait (atom 1100)]
    (fn [world]
      (if (= 0 (swap! wait dec))
        (let [brick
              (box 0.8 0.8 0.8 :mass 0.05
                                :position (Vector3f. -0.5 0 0.5)
                                :color (ColorRGBA/Red))
              bell (AudioNode. (asset-manager)
                               "Sounds/pure.wav" false)]
          (.play bell)
          (if debug?
            (.addControl
             brick
             (proxy [AbstractControl] []
               (controlUpdate [tpf]
                 (println-repl (.getWorldTranslation brick)))
               (controlRender [_ _]))))
          (add-element world brick))))))

(import com.aurellem.capture.Capture)

(defn test-integration 
  "Testing Everything!

   You will see an articulated hand fall onto the table. It has all
   senses including:
     - Vision, 4 channels
     - Hearing
     - Touch
     - Proprioceptoin
     - Muscle Tension

    Keys:
    <space> : fire ball"
  ([] (test-integration false))
  ([record?]
  (let [me (sphere 0.5 :color ColorRGBA/Blue :physical? false)

        base (File. "/home/r/proj/cortex/render/hand")
        
        
        creature (doto (load-blender-model hand) 
                   (body!))

        summon-ball (gen-summon-ball false)
        ;;;;;;;;;;;;  Sensors/Effectors  ;;;;;;;;;;;;;;;;;;;;;;;;;;;;
        touch (touch! creature)
        touch-display (view-touch)

        vision (vision! creature)
        vision-display (view-vision)

        hearing (hearing! creature)
        hearing-display (view-hearing)

        prop (proprioception! creature)
        prop-display (view-proprioception)

        control-script (motor-control-program
                        muscle-positions move-fingers)
        muscles (movement! creature)
        muscle-display (view-movement)
        ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

        fix-display (gen-fix-display)]
    (world
     (nodify [creature
              (box 10 2 10 :position (Vector3f. 0 -9 0)
                   :color ColorRGBA/Gray :mass 0)
              me])
     standard-debug-controls
    
     (fn [world]
       (.setTimer world (RatchetTimer. 60))
       (position-camera
        world (Vector3f. -0.13217318, 5.816415, -5.3089414)
        (Quaternion. 0.55685693, 0.0042774677, -0.0028673497, 0.83059245))

       (light-up-everything world)
       (enable-debug world)
       (add-camera! world
                    (add-eye! creature
                              (.getChild 
                               (.getChild creature "eyes") "eye"))
                    (comp (view-image) BufferedImage!))

       (if record?
         (Capture/captureVideo
          world (File. base "main")))
       (if record?
         (Capture/captureAudio
          world (File. base "main.wav"))))
     (fn [world tpf]
       (prop-display
        (prop)
        (if record? (File. base "proprio")))             
       (touch-display
        (map #(% (.getRootNode world)) touch)
        (if record? (File. base "touch")))
       (vision-display
        (map #(% world) vision)
        (if record? (File. base "vision")))
       (hearing-display
        (map #(% world) hearing)
        (if record? (File. base "hearing")))
       (muscle-display
        (control-script muscles)
        (if record? (File. base "muscle")))
         
       (summon-ball world)
       
       (.setLocalTranslation me (.getLocation (.getCamera world)))
       (fix-display world))))))

2.1 ImageMagick / ffmpeg

Just a bunch of calls to imagemagick to arrange the data that test-everything! produces.

(ns cortex.video.magick8
  (:import java.io.File)
  (:use clojure.java.shell))

(comment
;; list of touch targets
0  middle-11 
1  middle-21 
2  middle-31
3  pinky-11
4  pinky-21
5  pinky-31
6  pointer-11
7  pointer-21
8  pointer-31
9  ring-11
10 ring-21
11 ring-31
12 thumb-11
13 thumb-2.0011 


;; list of vision targets
0  :all
1  :green
2  :blue
3  :red 

;; list of proprio targets
0  middle-11 -> middle-21
1  middle-21 -> middle-31
2  thumb-11 -> thumb-2.0011
3  pointer-11 -> pointer-21
4  pointer-21 -> pointer-31
5  ring-21 -> ring-31
6  ring-11 -> ring-21
7  pinky-21 -> pinky-31
8  pinky-11 -> pinky-21
9  middle-11 -> palm1
10 pinky-11 -> palm1
11 palm1 -> pointer-11
12 palm1 -> ring-11
13 palm1 -> thumb-11


;; list of muscle targets
0  :pointer-2-e
1  :pointer-2-f
2  :thumb-1
3  :thumb-1
4  :pointer-1-e
5  :pointer-1-f
6  :thumb-2-e
7  :thumb-2-f
8  :middle-1-e
9  :middle-1-f
10 :pointer-3-f
11 :pointer-3-e
12 :middle-2-e
13 :middle-2-f
14 :middle-3-f
15 :middle-3-e
16 :pinky-2-e
17 :pinky-2-f
18 :pinky-3-f
19 :pinky-3-e
20 :ring-3-e
21 :ring-3-f
22 :ring-2-f
23 :ring-2-e
24 :ring-1-e
25 :ring-1-f
26 :thumb-1-e
27 :thumb-1-f
28 :pinky-1-f
29 :pinky-1-e
)

(def base (File. "/home/r/proj/cortex/render/hand"))

(defn prepare-muscle [muscle]
  ["(" muscle "-rotate" "90" "-scale" "15x60!" ")"])

(defn prepare-touch [touch]
  ["(" touch "-rotate" "180" ")"])

(defn generate-top-finger [tip-flexor tip-extensor tip
                           joint-2-3
                           mid-flexor mid-extensor mid
                           joint-1-2]
  ["("
   "-size" "113x357" "xc:transparent"
   (prepare-muscle tip-flexor)   "-geometry" "+0+7"    "-composite"
   (prepare-muscle tip-extensor) "-geometry" "+98+7"   "-composite"
   (prepare-touch tip)           "-geometry" "+18+0"   "-composite"

   joint-2-3                     "-geometry" "+32+79" "-composite"

   (prepare-muscle mid-flexor)   "-geometry" "+19+131"    "-composite"
   (prepare-muscle mid-extensor) "-geometry" "+80+131"   "-composite"
   (prepare-touch mid)           "-geometry" "+39+133"   "-composite"
   
   joint-1-2                     "-geometry" "+32+193" "-composite"
   ")"])

(defn file-names [#^File dir]
   (map #(.getCanonicalPath %) (next (sort (file-seq dir)))))

(defn file-groups [& paths]
  (apply (partial map list )
         (map (comp file-names #(File. base %))
              paths)))
  
(defn pinky []
  (file-groups
   "muscle/18"
   "muscle/19"
   "touch/5"
   "proprio/7"
   
   "muscle/17"
   "muscle/16"
   "touch/4"
   "proprio/8"
   
   "muscle/28"
   "muscle/29"
   "touch/3"
   "proprio/10"))

(defn ring []
  (file-groups
   "muscle/21"
   "muscle/20"
   "touch/11"
   "proprio/5"

   "muscle/22"
   "muscle/23"
   "touch/10"
   "proprio/6"

   "muscle/25"
   "muscle/24"
   "touch/9"
   "proprio/12"))

(defn middle []
  (file-groups
   "muscle/14"
   "muscle/15"
   "touch/2"
   "proprio/1"

   "muscle/13"
   "muscle/12"
   "touch/1"
   "proprio/0"

   "muscle/9"
   "muscle/8"
   "touch/0"
   "proprio/9"))

(defn pointer []
  (file-groups
   "muscle/10"
   "muscle/11"
   "touch/8"
   "proprio/4"

   "muscle/1"
   "muscle/0"
   "touch/7"
   "proprio/3"

   "muscle/5"
   "muscle/4"
   "touch/6"
   "proprio/11"))

(defn thumb []
  (file-groups
   "muscle/7"
   "muscle/6"
   "touch/13"
   "proprio/2"

   "muscle/27"
   "muscle/26"
   "muscle/3"
   "muscle/2"
   "touch/12"
   "proprio/13"))

(defn generate-finger
  [tip-flexor tip-extensor tip
   joint-2-3
   mid-flexor mid-extensor mid
   joint-1-2
   base-flexor base-extensor base
   joint-palm-1]
  ["("
   "-size" "113x357" "xc:transparent"
   (generate-top-finger
    tip-flexor  tip-extensor tip
    joint-2-3
    mid-flexor mid-extensor mid
    joint-1-2)                    "-geometry" "+0+0"      "-composite"
   (prepare-muscle base-flexor)   "-geometry" "+19+245"   "-composite"
   (prepare-muscle base-extensor) "-geometry" "+80+245"   "-composite"
   (prepare-touch base)           "-geometry" "+39+247"   "-composite"
   joint-palm-1                   "-geometry" "+32+307"   "-composite"
   ")"])

(defn generate-thumb
  [tip-flexor tip-extensor tip
   joint-1-2
   mid-flexor mid-extensor mid-flexor-2 mid-extensor-2 mid
   joint-palm-1]
  ["("
   "-size" "113x357" "xc:transparent"
   (generate-top-finger
    tip-flexor  tip-extensor tip
    joint-1-2
    mid-flexor mid-extensor mid
    joint-palm-1)                   "-geometry" "+0+0"     "-composite"
   (prepare-muscle mid-flexor-2)    "-geometry" "+2+131"   "-composite"
   (prepare-muscle mid-extensor-2)  "-geometry" "+100+131" "-composite"
   ")"])

(defn generate-hand
  [pinky-pieces
   ring-pieces
   middle-pieces
   pointer-pieces
   thumb-pieces]
  ["("
   "-size" "688x769" "xc:transparent"
   (apply generate-finger pinky-pieces)
   "-geometry" "+0+195" "-composite"
   (apply generate-finger ring-pieces)
   "-geometry" "+111+100" "-composite"
   (apply generate-finger middle-pieces)
   "-geometry" "+228+0"   "-composite"
   "(" (apply generate-thumb thumb-pieces) "-background" "#00000000"
   "-rotate" "45" ")"
   "-geometry" "+300+420" "-composite"
   (apply generate-finger pointer-pieces)
   "-geometry" "+350+96"  "-composite"
   ")"])

(defn generate-vision
  [all green blue red]
  ["("
   "-size" "204x192" "xc:transparent"
   all    "-geometry" "+0+0" "-composite"
   green  "-geometry" "+113+0" "-composite"
   blue   "-geometry" "+0+105" "-composite"
   red    "-geometry" "+113+105" "-composite"
   ")"])
  
(def test-muscle (File. base "muscle/0/0000000.png"))
(def test-proprio (File. base "proprio/0/0000000.png"))
(def test-tip (File. base "touch/2/0000000.png"))
(def test-mid (File. base "touch/0/0000000.png"))
(def test-vision (File. base "vision/0/0000000.png"))
(def test-hearing (File. base "hearing/0/0000000.png"))
(def test-main (File. base "main/0000000.png"))

(def test-target (File. base "output.png"))

(def background (File. base "background.png"))

(use 'clojure.java.shell)

(defn vision []
  (file-groups
   "vision/0"
   "vision/1"
   "vision/2"
   "vision/3"))

(defn hearing []
  (file-names (File. base "hearing/0")))

(defn main []
  (file-names (File. base "main")))

(defn targets [dest max]
  (map
   (comp #(.getCanonicalPath %)
         #(File. (str base dest (format "%07d.png" %))))
   (range max)))


(defn final-image [main [all red green blue] hearing
                   pinky ring middle pointer thumb target]
  (println target)
  (apply
     sh
     (flatten
      ["convert"
       (.getCanonicalPath background)
       
       (generate-hand pinky ring middle pointer thumb)
       "-geometry" "+809+22" "-composite"
       
       (generate-vision all red green blue)
       "-geometry" "+974+599" "-composite"

       hearing
       "-geometry" "+784+819" "-composite"

       main
       "-geometry" "+78+202" "-composite"
              
       target])))
  
(defn combine-files []
  (dorun
   (pmap final-image
         (main)
         (vision)
         (hearing)
         (pinky)
         (ring)
         (middle)
         (pointer)
         (thumb)
         (targets "/out/" (count (main))))))

(defn subtitles [] 
  (file-names (File. base "subs")))

(defn outs []
  (file-names (File. base "out")))


(defn mix-subtitles []
  (let [subs (subtitles)
        targets (targets "/out-subs/" (count subs))
        overlay (.getCanonicalPath (File. base "output.png"))]
    (dorun 
     (pmap
      (fn [sub target]
        (sh
         "convert"
         overlay
         sub "-geometry" "+0+0" "-composite"
         target))
      subs targets))))

(defn out-subtitles []
  (file-names (File. base "out-subs")))
                    

(defn insert-subtitles []
  (let [subtitles (out-subtitles)
        outs (outs)
        targets (targets
                 "/final/"
                 (+ (count outs) (count subtitles)))]
    (dorun
     (pmap
      #(sh "cp" %1 %2)
      (concat subtitles outs) targets))))

(defn generate-final []
  (combine-files)
   (mix-subtitles)
   (insert-subtitles))
cd /home/r/proj/cortex/render/hand

sox --ignore-length main.wav main-delayed.wav delay 24 

mogrify -resize 755x final/*
cd /home/r/proj/cortex/render/hand

ffmpeg -r 60 -i final/%07d.png -i main-delayed.wav -b:a 128k \
         -b:v 9000k -c:a libvorbis -c:v libtheora hand.ogg

3 Source Listing

Author: Robert McIntyre

Created: 2015-04-19 Sun 07:04

Emacs 24.4.1 (Org mode 8.3beta)

Validate