MyMorphing
In [1]:
import numpy as np
import matplotlib
import matplotlib.image as image
import matplotlib.pyplot as plt
from skimage.transform import PiecewiseAffineTransform, warp, resize
%matplotlib notebook
# NOTE: all “magic” options for backend plotting are: inline, notebook, and “external” (default)
# see http://ipython.readthedocs.io/en/stable/interactive/plotting.html for details
LineBuilder class for entering correspoinding points in each image¶
Note: left mouse clicks add polyline intervals, the right mouse click terminates the polyline, ‘DEL’ key removes the last click point¶
In [2]:
class LineBuilder:
def __init__(self, axes, line):
self.line = line
self.ax = axes
self.xs = list(line.get_xdata())
self.ys = list(line.get_ydata())
self.dashed_line = plt.Line2D((0,0),(0,0), color = line.get_c(), ls=’:’, lw=line.get_linewidth())
self.terminated = False
self.connect()
def connect(self):
# ‘connect to all the events we need’
self.cidpress = self.ax.figure.canvas.mpl_connect(‘button_press_event’, self.on_click)
self.cidmotion = self.ax.figure.canvas.mpl_connect(‘motion_notify_event’, self.on_motion)
self.cidkey = self.ax.figure.canvas.mpl_connect(‘key_press_event’, self.on_key_down)
def on_click(self, event):
if ((self.terminated==True) or (event.inaxes!=self.line.axes)): return
self.xs.append(event.xdata)
self.ys.append(event.ydata)
self.line.set_data(self.xs, self.ys)
#self.ax.add_patch(plt.Circle((event.xdata,event.ydata), radius=5, fc=’y’))
self.line.figure.canvas.draw()
if (event.button==3): self.terminated = True #right mouse click terminates the polyline
def on_motion(self, event):
if ((self.terminated==True) or (event.inaxes!=self.line.axes) or (len(self.xs)==0)): return
self.dashed_line.set_xdata([self.xs[-1],event.xdata])
self.dashed_line.set_ydata([self.ys[-1],event.ydata])
self.ax.add_line(self.dashed_line)
def on_key_down(self, e): # do the following when certain “key” is pressed
if (e.inaxes!=self.ax.axes): return
if e.key == ‘delete’ : # pressing ‘DEL’ key deletes the last click
if self.xs : self.xs.pop()
if self.ys : self.ys.pop()
self.line.set_data(self.xs, self.ys)
self.dashed_line.set_xdata([])
self.dashed_line.set_ydata([])
self.line.figure.canvas.draw()
self.terminated = False
def disconnect(self):
# ‘disconnect all the stored connection ids’
self.ax.figure.canvas.mpl_disconnect(self.cidmotion)
self.ax.figure.canvas.mpl_disconnect(self.cidpress)
self.ax.figure.canvas.mpl_disconnect(self.cidkey)
CrossDissolver class¶
Note: click on the left side to move back to image A, clicks on the right side to move forward to image B¶
In [3]:
# ASSUMPTION: images A and B should have the same size and the same data type (‘uint8’ {0,1,2,…,255} or ‘float’ [0,1])
class CrossDissolver:
def __init__(self, axes, imA, imB, dtype = ‘uint8’):
self.ax = axes
scaler = 1.0/255.0
if (dtype == ‘float’) : scaler = 1.0
self.A = scaler * np.array(imA) # generating array A with (RGB) float values in the range [0,1]
self.B = scaler * np.array(imB) # generating array B with (RGB) float values in the range [0,1]
self.t = 0.5
self.width = imA.shape[1]
self.text_pos = (self.width+20, 20)
self.cidpress = self.ax.figure.canvas.mpl_connect(‘button_press_event’, self.on_click)
self.showMix()
def on_click(self, event):
if (event.inaxes!=self.ax.axes): return
if (event.xdata>(self.width/2)): self.t = min(1.0,self.t+0.1)
if (event.xdata<(self.width/2)): self.t = max(0.0,self.t-0.1)
self.showMix()
def showMix(self):
self.ax.imshow((1.0-self.t)*self.A + self.t*self.B)
self.ax.text(self.text_pos[0], self.text_pos[1], 't = {:4.2f}'.format(self.t), color='y', backgroundcolor='gray')
In [4]:
im1 =image.imread("images/Fontanka1.jpg")
im2 =image.imread("images/Fontanka2.jpg")
im3 =image.imread("images/woman.png") # NOTE: images "woman.png" and "cheetah.png" have RGV values as floats in [0,1]
im4 =image.imread("images/cheetah.png")
fig = plt.figure(1,figsize = (12, 4))
ax12 = plt.subplot(121)
css12 = CrossDissolver(ax12,im1,im2) # instantiates CrossDissolver object
plt.title("cross-dissolving images 1 and 2")
ax34 = plt.subplot(122)
css34 = CrossDissolver(ax34,im3,resize(im4,im3.shape),'float') # we use resized image im4 to match the size of im3
plt.title("cross-dissolving images 3 and 4")
Out[4]:
Morphing¶
In [30]:
imA =image.imread(“images/woman.png”)
imB =image.imread(“images/cheetah.png”)
fig = plt.figure(2,figsize = (7, 5))
axA = plt.subplot(121)
polylineA = LineBuilder(axA, plt.plot([], [], ‘ro-‘)[0] ) # instantiates LineBuilder object
axA.imshow(imA)
plt.title(“image A”)
axB = plt.subplot(122)
polylineB = LineBuilder(axB, plt.plot([], [], ‘bo-‘)[0] ) # instantiates LineBuilder object
axB.imshow(imB)
plt.title(“image B”)
# plt.show()
# (Left) click on two images below to consecutively enter corresponding points (right clicks enter the last points)
# Press “DELETE” key to delete the last click point.
Out[30]:
Delaunay triangulation (2D mesh) over the points in each image¶
In [31]:
ptsA = polylineA.line.get_xydata()
ptsB = polylineB.line.get_xydata()
rowsA, colsA = imA.shape[0], imA.shape[1]
rowsB, colsB = imB.shape[0], imB.shape[1]
import matplotlib.tri as tri
triA = tri.Triangulation(ptsA[:,0],ptsA[:,1])
triB = tri.Triangulation(ptsB[:,0],ptsB[:,1])
plt.figure(3,figsize = (12, 8))
plt.subplot(121)
plt.axis((0, colsA, rowsA, 0))
plt.imshow(imA)
plt.triplot(triA, ‘ro-‘)
plt.title(‘Delaunay triangulation for A’)
plt.subplot(122)
plt.axis((0, colsB, rowsB, 0))
plt.imshow(imB)
plt.triplot(triB, ‘bo-‘)
plt.title(‘Delaunay triangulation for B’)
Out[31]:
Piece-wise Affine transformation between two meshes¶
In [32]:
pwAffAtoB = PiecewiseAffineTransform()
pwAffAtoB.estimate(ptsA, ptsB) # estimates piece-wise affine transformation from A to B
# Each (Delaunay) triangle ABC over ptsA (triangulation is scomputed inside)
# is mapped onto a triangle A’B’C’ over ptsB implied by correspondences between ptsA and ptsB
# A distinct affine transformation is computed for each triangle
Out[32]:
True
Warping images using piece-wise affine transformation computed above¶
NOTE 1: function “warp” uses “inverse map” as a (second) argument as it uses “inverse warping” to compute output intensities¶
NOTE 2: function “warp” generates images with RGB values as floats in range [0,1] (float is a result of interpolation)¶
In [33]:
AtoB = warp(imA, pwAffAtoB.inverse, output_shape=(rowsB, colsB))
BtoA = warp(imB, pwAffAtoB, output_shape=(rowsA, colsA)) # pwAffAtoB is the inverse for B->A transform, i.e. pwAffAtoB.inverse
fig = plt.figure(4,figsize = (12, 8))
axA = plt.subplot(121)
axA.imshow(AtoB)
plt.title(“A -> B”)
axA.plot(ptsB[:, 0], ptsB[:, 1], ‘ob’) # original click points in B are shown (blue circles) to compare accuracy of projection
axA.plot(pwAffAtoB(ptsA)[:, 0], pwAffAtoB(ptsA)[:, 1], ‘.y’)
axA.axis((0, colsB, rowsB, 0))
axB = plt.subplot(122)
axB.imshow(BtoA)
plt.title(“B -> A”)
axB.plot(ptsA[:, 0], ptsA[:, 1], ‘or’) # original click points in A are shown (red circles) to compare accuracy of projection
axB.plot(pwAffAtoB.inverse(ptsB)[:, 0], pwAffAtoB.inverse(ptsB)[:, 1], ‘.y’)
axB.axis((0, colsA, rowsA, 0))
plt.show()
Interpolated (intermediate) warps for steps $t\in [0,1]$¶
In [34]:
# This function computes a warp of image A onto image B based on corresponding points given inside two images (pA and pB)
# Optional argument is the step size T in [0,1] with default value T=1.0 (full step)
# It is also possible to specify the size of the output image frame (in any case, “warp” fills out only mapped triangles)
# The function returns a transformed image and a set of stransformed points (at step t)
def pwAff_T(pA, pB, A, B, T=1.0, shape = None):
# YOUR CODE SHOULD BE WRITTEN HERE (also change the return values)
# USE CAN USE FUNCTIONS PiecewiseAffineTransform() and warp()
# see documentation in http://scikit-image.org/docs/dev/api/skimage.transform.html
pT = (1-T) * pA + T * pB
a2b = PiecewiseAffineTransform()
a2b.estimate(pA, pT)
imgRe = warp(A, a2b.inverse, output_shape=shape)
return imgRe, pT # THIS RETURNED IMAGE IS WRONG, IT IS ONLY SCALED TO HAVE THE RIGHT shape
# NOTE!!! return image has RGB values as floats in interval [0,1] (property of functions “warp” or “resize”)
# ALTERNATIVELY, you can change this function to use precomputed transform pwAffAtoB instead of points pA and pB
# as long as you figure out how to “interpolate” the corresponding image warp for 0
In [35]:
t = 0.6
rowsT, colsT = round(rowsB + t*(rowsA-rowsB)), round(colsB + t*(colsA-colsB))
AtoB_t, pts_t = pwAff_T(ptsA,ptsB,imA,imB, t, shape=(rowsT, colsT))
fig = plt.figure(5,figsize = (12, 4))
plt.subplot(131)
plt.axis((0, colsA, rowsA, 0))
plt.imshow(imA)
plt.title(“image A (0.0)”)
plt.plot(ptsA[:, 0], ptsA[:, 1], ‘.y’)
plt.subplot(132)
plt.axis((0, colsT, rowsT, 0))
plt.imshow(AtoB_t)
plt.title(“A -> B ({:3.1f})”.format(t))
plt.plot(pts_t[:, 0], pts_t[:, 1], ‘.y’)
plt.subplot(133)
plt.axis((0, colsB, rowsB, 0))
plt.imshow(AtoB)
plt.title(“A -> B (1.0)”)
plt.plot(pwAffAtoB(ptsA)[:, 0], pwAffAtoB(ptsA)[:, 1], ‘.y’)
plt.show()
# NOTE, the provided implementation for pwAff_T does not give the right intermediate warp results (middle image)
Cross-dissolving registered images A and B after warping them to the same intermediate frame $t\in [0,1]$¶
In [36]:
t = 0.6
# estimating common size for the intermediate frame
rowsT, colsT = round(rowsA + t*(rowsB-rowsA)), round(colsA + t*(colsB-colsA))
AtoB_t, pA_t = pwAff_T(ptsA,ptsB,imA,imB,t, shape = (rowsT,colsT))
BtoA_t, pB_t = pwAff_T(ptsB,ptsA,imB,imA,1.0-t, shape = (rowsT,colsT))
fig = plt.figure(6,figsize = (12, 8))
plt.subplot(221)
plt.axis((0, colsT, rowsT, 0))
plt.imshow(AtoB_t)
plt.title(“A -> B ({:3.1f})”.format(t))
plt.plot(pA_t[:, 0], pA_t[:, 1], ‘.r’)
plt.subplot(222)
plt.axis((0, colsT, rowsT, 0))
plt.imshow(BtoA_t)
plt.title(“B -> A ({:3.1f})”.format(1.0-t))
plt.plot(pB_t[:, 0], pB_t[:, 1], ‘.b’)
ax3 = plt.subplot(223)
cf3 = CrossDissolver(ax3,AtoB_t,BtoA_t,’float’) # NOTE: pwAff_T generated float images
plt.title(“A <-> B cross-dissolve”)
ax4 = plt.subplot(224)
plt.axis((0, colsT, rowsT, 0))
cf4 = CrossDissolver(ax4,AtoB_t,BtoA_t,’float’) # NOTE: pwAff_T generated float images
plt.plot(pA_t[:, 0], pA_t[:, 1], ‘ob’) # both points pA_t and pB_t are shown
plt.plot(pB_t[:, 0], pB_t[:, 1], ‘.r’) # to validate registration at frame t
plt.title(“cross-dissolve (with points)”)
plt.show()
# NOTE: for correctly implemented pwAff_T, red and blue dots at the top images should match with the selected featrues,
# red and blue points at the botton right image should coinside and the images should be registered when cross-dissolved
In [37]:
pA_t
Out[37]:
array([[ 160.25791763, 3.31533122],
[ 88.66768721, 33.14814228],
[ 24.51893145, 73.99717454],
[ 50.97570565, 177.86758929],
[ 158.20188076, 227.40510081],
[ 255.85754896, 180.56030818],
[ 296.24224942, 73.0421515 ],
[ 236.66989919, 34.88583813],
[ 162.7052909 , 2.99699021]])
Morpher class (morphing two images)¶
Note: click on the left side to move back to image A, clicks on the right side to move forward to image B¶
In [38]:
pB_t
Out[38]:
array([[ 160.25791763, 3.31533122],
[ 88.66768721, 33.14814228],
[ 24.51893145, 73.99717454],
[ 50.97570565, 177.86758929],
[ 158.20188076, 227.40510081],
[ 255.85754896, 180.56030818],
[ 296.24224942, 73.0421515 ],
[ 236.66989919, 34.88583813],
[ 162.7052909 , 2.99699021]])
In [39]:
# The Morpher generates intermediate steps T in [0,1] morphing image A onto B
# NOTE: images imA and imB do NOT have to be of the same size
class Morpher:
def __init__(self, axes, ptsA, ptsB, imA, imB, shape = None, showPts = False):
self.ax = axes
self.A = imA
self.B = imB
self.ptsA = ptsA
self.ptsB = ptsB
self.T = 0.5
self.showPts = showPts
self.morph_shape = shape
if (shape==None): self.morph_shape = (max(imA.shape[0],imB.shape[0]), max(imA.shape[1],imB.shape[1]))
self.width = self.morph_shape[1]
self.text_pos = (self.width+20, 20)
self.cidpress = self.ax.figure.canvas.mpl_connect(‘button_press_event’, self.on_click)
self.ax.axis((0, self.morph_shape[1], self.morph_shape[0], 0))
self.showMorph()
def on_click(self, event):
if (event.inaxes!=self.ax.axes): return
if (event.xdata>(self.width/2)): self.T = min(1.0,self.T+0.1)
if (event.xdata<(self.width/2)): self.T = max(0.0,self.T-0.1)
self.showMorph()
def showMorph(self):
# WRITE YOUR CODE TO SHOW MORPH BETWEEN IMAGES A TO B (REPLACING CROSS-DISSOLVE CODE BELOW) FOR GIVEN self.T
# NOTE: YOU SHOULD SHOW POINTS (IN BLUE AND RED) ptsA and ptsB MAPPED ONTO SELECTED FRAME 0
# Click anywhere on the left side to move closer to imA and click on the right side to move closer to imB
# Morpher object increments or deincrements T after such clicks.
Out[41]:
In [ ]:
In [ ]: