Motion Interpolation CG Proxies

When Disney’s Paperman came out it discovered a new technique. This was using 3d vectors generated from animated geometry to create in-between frames of 2d drawn images or Motion Vector Inbetweening

Several VFX companies have started using this technique to dramatically cut the cost of 3d renders. By rendering on less frames i.e. 4, 6 or 8’s, a render that would take a week, takes a day. The quality might not be perfect with softening and the occasional jaggies. But this quality is more than good enough to make creative decisions with or show to the client as work in progress. Also because there are ‘real’ rather than motion vector frames rendered on every 4,6 or 8, quality control on details such as noise, sharpness and motion blur can be evaluated on these frames. Having this feature built into a 3d render engine would sure help a lot of filmmakers out!

I put aside a day to try and work this out in Nuke and this image bellow is as far as i got. For non occluded objects that have similar motion vectors it works well. But as you can see from the edges of the mauve square bellow. I still have a problem that occluded objects in depth from the camera that have different motion vectors and are not resolving correctly. I have a few ideas like blending motion vectors and creating edge mattes from contrast areas of the motion vectors, but I need some time, a nuke licence and help to solve the problem. In addition to working out this basic issue, the second problem that will need to be solved is how to deal with motion blur. The nuke code is below to copy and paste: If anyone with a grasp for vector maths can improve this please contact me!

In case you haven’t seen ‘Paperman’ watch it now, it’s wonderful!

EDIT: A RnD friend is currently helping me on this one.. Hope to get a result soon!

set cut_paste_input [stack 0]
version 9.0 v2
Camera2 {
inputs 0
translate {0 0 5}
version 28
name Camera1
selected true
xpos 4528
ypos 249
}
CheckerBoard2 {
inputs 0
name CheckerBoard2
selected true
xpos 4685
ypos -80
}
Dot {
name Dot1
selected true
xpos 4719
ypos 22
}
set N79a76820 [stack 0]
Grade {
multiply {0 1 0 1}
name Grade1
selected true
xpos 4740
ypos 63
}
Sphere {
translate {{curve x9 -1.649999976 x31.81542969 1.196943998 x54 -1.558774948} {curve x9 0 x31 -1.089066982 x54 0} {curve x1009 -6.159999847 x1054 -6.159999847}}
name Sphere2
selected true
xpos 4740
ypos 154
}
push $N79a76820

Grade {
multiply {1 0 1 1}
name Grade2
selected true
xpos 4623
ypos 66
}
Cube {
translate {{curve x9 0} {curve x9 0} {curve x1009 -3.799999952}}
name Cube2
selected true
xpos 4623
ypos 150
}
MergeGeo {
inputs 2
name MergeGeo2
selected true
xpos 4692
ypos 207
}
push $cut_paste_input
Reformat {
name Reformat1
selected true
xpos 4811
ypos 221
}
ScanlineRender {
inputs 3
overscan 200
motion_vectors_type distance
output_shader_vectors true
name ScanlineRender1
selected true
xpos 4692
ypos 282
}
Dot {
name Dot2
selected true
xpos 4726
ypos 361
}
set N79aa5800 [stack 0]
Dot {
name Dot3
selected true
xpos 4727
ypos 929
}
set N79aa8000 [stack 0]
Dot {
name Dot4
selected true
xpos 4867
ypos 929
}
TimeOffset {
time_offset 1
time “”
name TimeOffset1
selected true
xpos 4833
ypos 1295
}
push $N79aa5800
Dot {
name Dot5
selected true
xpos 4598
ypos 361
}
set N79ab0a00 [stack 0]
Shuffle {
in motion
name Shuffle2
selected true
xpos 4443
ypos 358
}
Dot {
name Dot6
selected true
xpos 4477
ypos 578
}
set N79ab9cc0 [stack 0]
Dot {
name Dot7
selected true
xpos 4315
ypos 578
}
set N79abc130 [stack 0]
Dot {
name Dot8
selected true
xpos 4237
ypos 578
}
set N79abe680 [stack 0]
Dot {
name Dot9
selected true
xpos 4156
ypos 578
}
set N79ac0bd0 [stack 0]
Dot {
name Dot10
selected true
xpos 3951
ypos 578
}
Expression {
temp_expr0 !r==0&G==0?1:0
expr0 r==0&&r==0?0:1
expr1 r==0&&r==0?0:1
expr2 r==0&&r==0?0:1
channel3 alpha
expr3 r==0&&r==0?0:1
name Expression2
selected true
xpos 3917
ypos 703
}
set N79ac5670 [stack 0]
Dot {
name Dot11
selected true
xpos 3841
ypos 706
}
TimeOffset {
time_offset 1
time “”
name TimeOffset3
selected true
xpos 3807
ypos 779
}
push $N79ac5670
Merge2 {
inputs 2
operation from
name Merge1
selected true
xpos 3917
ypos 779
}
set N79ad23c0 [stack 0]
Shuffle {
green red
blue red
alpha red
name Shuffle3
selected true
xpos 3917
ypos 838
}
Clamp {
name Clamp1
selected true
xpos 3917
ypos 1161
}
push $N79abe680
Dot {
name Dot12
selected true
xpos 4237
ypos 1109
}
push $N79ad23c0
Dot {
name Dot13
selected true
xpos 4027
ypos 782
}
Multiply {
value -1
name Multiply1
selected true
xpos 3991
ypos 866
}
Clamp {
name Clamp2
selected true
xpos 3991
ypos 912
}
push $N79ac0bd0
Dot {
name Dot14
selected true
xpos 4156
ypos 705
}
Merge2 {
inputs 2
operation plus
name Merge2
selected true
xpos 4122
ypos 918
}
Shuffle {
green red
blue red
alpha red
name Shuffle4
selected true
xpos 4122
ypos 1027
}
push $N79abc130
TimeOffset {
time_offset 1
time “”
name TimeOffset6
selected true
xpos 4281
ypos 954
}
Add {
inputs 1+1
value {0 0.4 0 0}
name Add1
selected true
xpos 4281
ypos 1021
}
Merge2 {
inputs 2
operation from
name Merge3

selected true
xpos 4281
ypos 1106
}
Merge2 {
inputs 2
operation stencil
bbox B
maskChannelInput -rgba.alpha
name Merge4
selected true
xpos 4281
ypos 1167
}
push $N79ab9cc0
Merge2 {
inputs 2
operation plus
invert_mask true
name Merge5
selected true
xpos 4443
ypos 1167
}
Expression {
temp_name0 aspect
temp_expr0 1/((width/height))
temp_name1 ((g*2)/(height/2))
temp_expr1 ((y+0.5)/height)*(1+((g)/height))
temp_expr2 ((r)/(width))*4.
temp_expr3 ((g)/(height))*4.75
expr0 ((r)/(width))*4.
expr1 ((g)/(height))*4
channel3 {rgba.alpha none none rgba.alpha}
expr3 a
name Expression3
selected true
xpos 4443
ypos 1289
}
push $N79ab0a00
Expression {
temp_expr0 ((x+0.5)/width)*(1+((r)/width))
temp_expr1 ((y+0.5)/height)*(1+((g)/height))
temp_expr2 ((x)/width)-((((r*2)))/width)
temp_expr3 ((y)/height)-((((g*2)))/(height))
expr0 ((x+0.5)/width)
expr1 ((y+0.5)/height)
channel3 {rgba.alpha none none rgba.alpha}
expr3 a
name Expression4
selected true
xpos 4564
ypos 1077
}
Merge2 {
inputs 2
operation plus
name Merge6
selected true
xpos 4564
ypos 1289
}
push $N79aa8000
STMap {
inputs 2
uv rgba
blur -rgba.alpha
blur_scale 0
maskChannel -rgba.alpha
name STMap1
selected true
xpos 4696
ypos 1285
}
Viewer {
frame 42
frame_range 1-100
name Viewer1
selected true
xpos 4696
ypos 1317
}