Integration
aurellem ☉
1 Integration
2 Generating the Video
(ns cortex.test.integration "let's play!" {:author "Robert McIntyre"} (:use (cortex world util import body sense hearing touch vision proprioception movement)) (:import (com.jme3.math ColorRGBA Vector3f)) (:import java.io.File) (:import com.jme3.audio.AudioNode) (:import com.aurellem.capture.RatchetTimer)) (dorun (cortex.import/mega-import-jme3)) ;;(rlm.rlm-commands/help) (def hand "Models/test-creature/hand.blend") (def output-base (File. "/home/r/proj/cortex/render/hand"))
For this demonstration I have to manually drive the muscles of the hand. I do this by creating a little mini-language to describe simulated muscle contraction.
(defn motor-control-program "Create a function which will execute the motor script" [muscle-positions script] (let [current-frame (atom -1) keyed-script (group-by first script) current-forces (atom {}) ] (fn [effectors] (let [indexed-effectors (vec effectors)] (dorun (for [[_ part force] (keyed-script (swap! current-frame inc))] (swap! current-forces (fn [m] (assoc m part force))))) (doall (map (fn [effector power] (effector (int power))) effectors (map #(@current-forces % 0) muscle-positions))))))) (def muscle-positions [:pointer-2-e :pointer-2-f :thumb-1 :thumb-1 :pointer-1-e :pointer-1-f :thumb-2-e :thumb-2-f :middle-1-e :middle-1-f :pointer-3-f :pointer-3-e :middle-2-e :middle-2-f :middle-3-f :middle-3-e :pinky-2-e :pinky-2-f :pinky-3-f :pinky-3-e :ring-3-e :ring-3-f :ring-2-f :ring-2-e :ring-1-e :ring-1-f :thumb-1-e :thumb-1-f :pinky-1-f :pinky-1-e]) (def full 9001) ;; Choreography: ;; Let the hand fall palm-up ;; it curls its phalanges, starting with the pinky. ;; it lets its phalanges fall back down. ;; block falls down onto the hand, accompanied by a sound. The block ;; can be seen by the hand's eye. ;; hand FORCEFULLY catapults the block so that it hits the camera. ;; the syntax here is [keyframe body-part force] (def move-fingers [[300 :pinky-3-f 50] [320 :pinky-2-f 80] [340 :pinky-1-f 100] [310 :ring-3-f 100] [330 :ring-2-f 120] [350 :ring-1-f 140] [330 :middle-3-f 120] [340 :middle-2-f 120] [360 :middle-1-f 30] [350 :pointer-3-f 120] [360 :pointer-2-f 120] [380 :pointer-1-f 30] [800 :pinky-3-f 0] [800 :pinky-2-f 0] [800 :pinky-1-f 0] [800 :ring-3-f 0] [800 :ring-2-f 0] [800 :ring-1-f 0] [800 :middle-3-f 0] [800 :middle-2-f 0] [800 :middle-1-f 0] [800 :pointer-3-f 0] [800 :pointer-2-f 0] [800 :pointer-1-f 0] [800 :pinky-3-e 50] [800 :pinky-2-e 80] [800 :pinky-1-e 100] [800 :ring-3-e 100] [800 :ring-2-e 120] [800 :ring-1-e 140] [800 :middle-3-e 120] [800 :middle-2-e 120] [800 :middle-1-e 30] [800 :pointer-3-e 120] [800 :pointer-2-e 120] [800 :pointer-1-e 30] [870 :pinky-3-e 0] [870 :pinky-2-e 0] [870 :pinky-1-e 0] [870 :ring-3-e 0] [870 :ring-2-e 0] [870 :ring-1-e 0] [870 :middle-3-e 0] [870 :middle-2-e 0] [870 :middle-1-e 0] [870 :pointer-3-e 0] [870 :pointer-2-e 0] [870 :pointer-1-e 0] [1500 :pointer-1-f full] [1500 :pointer-2-f full] [1500 :pointer-3-f full] [1500 :middle-1-f full] [1500 :middle-2-f full] [1500 :middle-3-f full] [1510 :pointer-1-f 0] [1510 :pointer-2-f 0] [1510 :pointer-3-f 0] [1510 :middle-1-f 0] [1510 :middle-2-f 0] [1510 :middle-3-f 0] ]) (defn gen-summon-ball [debug?] (let [wait (atom 1100)] (fn [world] (if (= 0 (swap! wait dec)) (let [brick (box 0.8 0.8 0.8 :mass 0.05 :position (Vector3f. -0.5 0 0.5) :color (ColorRGBA/Red)) bell (AudioNode. (asset-manager) "Sounds/pure.wav" false)] (.play bell) (if debug? (.addControl brick (proxy [AbstractControl] [] (controlUpdate [tpf] (println-repl (.getWorldTranslation brick))) (controlRender [_ _])))) (add-element world brick)))))) (import com.aurellem.capture.Capture) (defn test-integration "Testing Everything! You will see an articulated hand fall onto the table. It has all senses including: - Vision, 4 channels - Hearing - Touch - Proprioceptoin - Muscle Tension Keys: <space> : fire ball" ([] (test-integration false)) ([record?] (let [me (sphere 0.5 :color ColorRGBA/Blue :physical? false) base (File. "/home/r/proj/cortex/render/hand") creature (doto (load-blender-model hand) (body!)) summon-ball (gen-summon-ball false) ;;;;;;;;;;;; Sensors/Effectors ;;;;;;;;;;;;;;;;;;;;;;;;;;;; touch (touch! creature) touch-display (view-touch) vision (vision! creature) vision-display (view-vision) hearing (hearing! creature) hearing-display (view-hearing) prop (proprioception! creature) prop-display (view-proprioception) control-script (motor-control-program muscle-positions move-fingers) muscles (movement! creature) muscle-display (view-movement) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; fix-display (gen-fix-display)] (world (nodify [creature (box 10 2 10 :position (Vector3f. 0 -9 0) :color ColorRGBA/Gray :mass 0) me]) standard-debug-controls (fn [world] (.setTimer world (RatchetTimer. 60)) (position-camera world (Vector3f. -0.13217318, 5.816415, -5.3089414) (Quaternion. 0.55685693, 0.0042774677, -0.0028673497, 0.83059245)) (light-up-everything world) (enable-debug world) (add-camera! world (add-eye! creature (.getChild (.getChild creature "eyes") "eye")) (comp (view-image) BufferedImage!)) (if record? (Capture/captureVideo world (File. base "main"))) (if record? (Capture/captureAudio world (File. base "main.wav")))) (fn [world tpf] (prop-display (prop) (if record? (File. base "proprio"))) (touch-display (map #(% (.getRootNode world)) touch) (if record? (File. base "touch"))) (vision-display (map #(% world) vision) (if record? (File. base "vision"))) (hearing-display (map #(% world) hearing) (if record? (File. base "hearing"))) (muscle-display (control-script muscles) (if record? (File. base "muscle"))) (summon-ball world) (.setLocalTranslation me (.getLocation (.getCamera world))) (fix-display world))))))
2.1 ImageMagick / ffmpeg
Just a bunch of calls to imagemagick to arrange the data that
test-everything! produces.
(ns cortex.video.magick8 (:import java.io.File) (:use clojure.java.shell)) (comment ;; list of touch targets 0 middle-11 1 middle-21 2 middle-31 3 pinky-11 4 pinky-21 5 pinky-31 6 pointer-11 7 pointer-21 8 pointer-31 9 ring-11 10 ring-21 11 ring-31 12 thumb-11 13 thumb-2.0011 ;; list of vision targets 0 :all 1 :green 2 :blue 3 :red ;; list of proprio targets 0 middle-11 -> middle-21 1 middle-21 -> middle-31 2 thumb-11 -> thumb-2.0011 3 pointer-11 -> pointer-21 4 pointer-21 -> pointer-31 5 ring-21 -> ring-31 6 ring-11 -> ring-21 7 pinky-21 -> pinky-31 8 pinky-11 -> pinky-21 9 middle-11 -> palm1 10 pinky-11 -> palm1 11 palm1 -> pointer-11 12 palm1 -> ring-11 13 palm1 -> thumb-11 ;; list of muscle targets 0 :pointer-2-e 1 :pointer-2-f 2 :thumb-1 3 :thumb-1 4 :pointer-1-e 5 :pointer-1-f 6 :thumb-2-e 7 :thumb-2-f 8 :middle-1-e 9 :middle-1-f 10 :pointer-3-f 11 :pointer-3-e 12 :middle-2-e 13 :middle-2-f 14 :middle-3-f 15 :middle-3-e 16 :pinky-2-e 17 :pinky-2-f 18 :pinky-3-f 19 :pinky-3-e 20 :ring-3-e 21 :ring-3-f 22 :ring-2-f 23 :ring-2-e 24 :ring-1-e 25 :ring-1-f 26 :thumb-1-e 27 :thumb-1-f 28 :pinky-1-f 29 :pinky-1-e ) (def base (File. "/home/r/proj/cortex/render/hand")) (defn prepare-muscle [muscle] ["(" muscle "-rotate" "90" "-scale" "15x60!" ")"]) (defn prepare-touch [touch] ["(" touch "-rotate" "180" ")"]) (defn generate-top-finger [tip-flexor tip-extensor tip joint-2-3 mid-flexor mid-extensor mid joint-1-2] ["(" "-size" "113x357" "xc:transparent" (prepare-muscle tip-flexor) "-geometry" "+0+7" "-composite" (prepare-muscle tip-extensor) "-geometry" "+98+7" "-composite" (prepare-touch tip) "-geometry" "+18+0" "-composite" joint-2-3 "-geometry" "+32+79" "-composite" (prepare-muscle mid-flexor) "-geometry" "+19+131" "-composite" (prepare-muscle mid-extensor) "-geometry" "+80+131" "-composite" (prepare-touch mid) "-geometry" "+39+133" "-composite" joint-1-2 "-geometry" "+32+193" "-composite" ")"]) (defn file-names [#^File dir] (map #(.getCanonicalPath %) (next (sort (file-seq dir))))) (defn file-groups [& paths] (apply (partial map list ) (map (comp file-names #(File. base %)) paths))) (defn pinky [] (file-groups "muscle/18" "muscle/19" "touch/5" "proprio/7" "muscle/17" "muscle/16" "touch/4" "proprio/8" "muscle/28" "muscle/29" "touch/3" "proprio/10")) (defn ring [] (file-groups "muscle/21" "muscle/20" "touch/11" "proprio/5" "muscle/22" "muscle/23" "touch/10" "proprio/6" "muscle/25" "muscle/24" "touch/9" "proprio/12")) (defn middle [] (file-groups "muscle/14" "muscle/15" "touch/2" "proprio/1" "muscle/13" "muscle/12" "touch/1" "proprio/0" "muscle/9" "muscle/8" "touch/0" "proprio/9")) (defn pointer [] (file-groups "muscle/10" "muscle/11" "touch/8" "proprio/4" "muscle/1" "muscle/0" "touch/7" "proprio/3" "muscle/5" "muscle/4" "touch/6" "proprio/11")) (defn thumb [] (file-groups "muscle/7" "muscle/6" "touch/13" "proprio/2" "muscle/27" "muscle/26" "muscle/3" "muscle/2" "touch/12" "proprio/13")) (defn generate-finger [tip-flexor tip-extensor tip joint-2-3 mid-flexor mid-extensor mid joint-1-2 base-flexor base-extensor base joint-palm-1] ["(" "-size" "113x357" "xc:transparent" (generate-top-finger tip-flexor tip-extensor tip joint-2-3 mid-flexor mid-extensor mid joint-1-2) "-geometry" "+0+0" "-composite" (prepare-muscle base-flexor) "-geometry" "+19+245" "-composite" (prepare-muscle base-extensor) "-geometry" "+80+245" "-composite" (prepare-touch base) "-geometry" "+39+247" "-composite" joint-palm-1 "-geometry" "+32+307" "-composite" ")"]) (defn generate-thumb [tip-flexor tip-extensor tip joint-1-2 mid-flexor mid-extensor mid-flexor-2 mid-extensor-2 mid joint-palm-1] ["(" "-size" "113x357" "xc:transparent" (generate-top-finger tip-flexor tip-extensor tip joint-1-2 mid-flexor mid-extensor mid joint-palm-1) "-geometry" "+0+0" "-composite" (prepare-muscle mid-flexor-2) "-geometry" "+2+131" "-composite" (prepare-muscle mid-extensor-2) "-geometry" "+100+131" "-composite" ")"]) (defn generate-hand [pinky-pieces ring-pieces middle-pieces pointer-pieces thumb-pieces] ["(" "-size" "688x769" "xc:transparent" (apply generate-finger pinky-pieces) "-geometry" "+0+195" "-composite" (apply generate-finger ring-pieces) "-geometry" "+111+100" "-composite" (apply generate-finger middle-pieces) "-geometry" "+228+0" "-composite" "(" (apply generate-thumb thumb-pieces) "-background" "#00000000" "-rotate" "45" ")" "-geometry" "+300+420" "-composite" (apply generate-finger pointer-pieces) "-geometry" "+350+96" "-composite" ")"]) (defn generate-vision [all green blue red] ["(" "-size" "204x192" "xc:transparent" all "-geometry" "+0+0" "-composite" green "-geometry" "+113+0" "-composite" blue "-geometry" "+0+105" "-composite" red "-geometry" "+113+105" "-composite" ")"]) (def test-muscle (File. base "muscle/0/0000000.png")) (def test-proprio (File. base "proprio/0/0000000.png")) (def test-tip (File. base "touch/2/0000000.png")) (def test-mid (File. base "touch/0/0000000.png")) (def test-vision (File. base "vision/0/0000000.png")) (def test-hearing (File. base "hearing/0/0000000.png")) (def test-main (File. base "main/0000000.png")) (def test-target (File. base "output.png")) (def background (File. base "background.png")) (use 'clojure.java.shell) (defn vision [] (file-groups "vision/0" "vision/1" "vision/2" "vision/3")) (defn hearing [] (file-names (File. base "hearing/0"))) (defn main [] (file-names (File. base "main"))) (defn targets [dest max] (map (comp #(.getCanonicalPath %) #(File. (str base dest (format "%07d.png" %)))) (range max))) (defn final-image [main [all red green blue] hearing pinky ring middle pointer thumb target] (println target) (apply sh (flatten ["convert" (.getCanonicalPath background) (generate-hand pinky ring middle pointer thumb) "-geometry" "+809+22" "-composite" (generate-vision all red green blue) "-geometry" "+974+599" "-composite" hearing "-geometry" "+784+819" "-composite" main "-geometry" "+78+202" "-composite" target]))) (defn combine-files [] (dorun (pmap final-image (main) (vision) (hearing) (pinky) (ring) (middle) (pointer) (thumb) (targets "/out/" (count (main)))))) (defn subtitles [] (file-names (File. base "subs"))) (defn outs [] (file-names (File. base "out"))) (defn mix-subtitles [] (let [subs (subtitles) targets (targets "/out-subs/" (count subs)) overlay (.getCanonicalPath (File. base "output.png"))] (dorun (pmap (fn [sub target] (sh "convert" overlay sub "-geometry" "+0+0" "-composite" target)) subs targets)))) (defn out-subtitles [] (file-names (File. base "out-subs"))) (defn insert-subtitles [] (let [subtitles (out-subtitles) outs (outs) targets (targets "/final/" (+ (count outs) (count subtitles)))] (dorun (pmap #(sh "cp" %1 %2) (concat subtitles outs) targets)))) (defn generate-final [] (combine-files) (mix-subtitles) (insert-subtitles))
cd /home/r/proj/cortex/render/hand
sox --ignore-length main.wav main-delayed.wav delay 24
mogrify -resize 755x final/*
cd /home/r/proj/cortex/render/hand ffmpeg -r 60 -i final/%07d.png -i main-delayed.wav -b:a 128k \ -b:v 9000k -c:a libvorbis -c:v libtheora hand.ogg