Mosaic Filter Recovery

recover.py

#!/usr/bin/env python3
# Dual license
# CC0 1.0 Universal https://creativecommons.org/publicdomain/zero/1.0/
# Zero-Clause BSD https://opensource.org/license/0bsd
# To the extent possible under law, Blackle Mori has waived all copyright and related or neighboring rights to this work.

import csv
import numpy as np
from PIL import Image

def generate_grid(N, M):
	x, y = np.meshgrid(range(N), range(M))
	grid = np.stack((x.ravel(), y.ravel()), axis=-1)
	return grid

def to_barycentric(point, tri_p1, tri_p2, tri_p3):
	mat = np.array([tri_p1 - tri_p3, tri_p2 - tri_p3]).T
	inv_mat = np.linalg.inv(mat)
	bary_coords = np.dot(inv_mat, point - tri_p3)
	bary_coords = np.array([bary_coords[0], bary_coords[1], 1 - bary_coords[0] - bary_coords[1]])
	inside = np.all(bary_coords >= 0) and np.all(bary_coords <= 1)
	return inside, bary_coords

def to_cartesian(bary_coords, tri_p1, tri_p2, tri_p3):
	return bary_coords[0] * tri_p1 + bary_coords[1] * tri_p2 + bary_coords[2] * tri_p3

def to_plane(point, tl, tr, bl, br):
	# it is easy to map a point from one triangle to another using barycentric coordinates
	# a plane is less easy
	# here, I take adavantage of the fact that a plane is made of two triangles
	# however, there are two ways to build the plane using two triangles
	# so I map the point with both possible triangulations, and take the average of each
	rtl = np.array([1,0])
	rtr = np.array([1,1])
	rbl = np.array([0,0])
	rbr = np.array([0,1])

	inside_count = 0
	converted = np.array([0,0],dtype=np.float32)

	def try_remap(tri_p1, tri_p2, tri_p3, rtri_p1, rtri_p2, rtri_p3):
		nonlocal converted, inside_count
		inside, mapped = to_barycentric(point, tri_p1, tri_p2, tri_p3)
		if inside:
			inside_count += 1
			converted += to_cartesian(mapped, rtri_p1, rtri_p2, rtri_p3)

	try_remap(tl,tr,br, rtl,rtr,rbr)
	try_remap(tl,tr,bl, rtl,rtr,rbl)
	try_remap(bl,br,tr, rbl,rbr,rtr)
	try_remap(bl,br,tl, rbl,rbr,rtl)

	if inside_count == 0:
		return None
	return converted/inside_count


def load_tracking(filename):
	tracking = {}
	with open(filename, 'r') as file:
		reader = csv.reader(file)
		for row in reader:
			key = int(row[0])
			value = np.array([float(row[1]), float(row[2])])
			tracking[key] = value
	return tracking

# export your tracks with:
# https://blender.stackexchange.com/a/197278/6294
# printFrameNums=True
# relativeCoords=True
def load_track(trackname):
	return load_tracking(f"trackcorrupted_Camera_tr_{trackname}.csv")

image_width = 1080
image_height = 1920
image_size = np.array([image_width,image_height])

mosaic_width=20
mosaic_height=20
mosaic_size = np.array([mosaic_width,mosaic_height])

# ABR is the bottom right of the mosaic
# ATL is the top left of the mosaic
array_br = list(load_track("ABR").values())[0]
array_tl = list(load_track("ATL").values())[0]
array_bl = np.array([array_br[0], array_tl[1]])
array_tr = np.array([array_tl[0], array_br[1]])

array_range = (array_bl-array_tr)

array_count = np.round(array_range*image_size/mosaic_size)+1
array_grid = generate_grid(int(array_count[0]), int(array_count[1]))
array_grid_imgcoords = array_grid/(array_count-1)*array_range+array_tr
print(array_grid)

tracks_TL = load_track("TL")
tracks_TR = load_track("TR")
tracks_BL = load_track("BL")
tracks_BR = load_track("BR")

start_frame=31
end_frame=91

out_width = 5000
out_height = 5000
out_size = np.array([out_width, out_height])
out_image = np.zeros((out_width, out_height, 4), dtype=np.uint8)

overlap_count = 0
for i in range(start_frame,end_frame+1):
	frame = Image.open(f"corrupted_frames/frame_{i:03}.png")
	tl = tracks_TL[i]
	tr = tracks_TR[i]
	bl = tracks_BL[i]
	br = tracks_BR[i]

	for coord, coord_img in zip(array_grid, array_grid_imgcoords):
		coord_pix = coord_img*image_size
		pix = frame.getpixel((int(coord_pix[0]), image_height-int(coord_pix[1])-1))
		plane = to_plane(coord_img, tl, tr, bl, br)
		if plane is None:
			continue

		plane *= out_size
		if out_image[int(plane[0])][int(plane[1])].all(0):
			overlap_count+=1
		out_image[int(plane[0])][int(plane[1])] = (pix[0], pix[1], pix[2], 255)

print(f"overlap: {overlap_count}")
output_image = Image.fromarray(out_image)
output_image.save(f"output.png")
# open output.png in gnu-imp and value propagate until the opaque pixels take up their entire neighbourhood