emotes/tools/capture/render.py

"""Renders model points through the cameras capture.py used, and scores them against captured masks."""
import json
from pathlib import Path

import numpy as np
from PIL import Image
from scipy.ndimage import distance_transform_edt

# Model space (Y down, facing -Z) to the viewer's world (Y up, facing +Z): Minecraft's scale(-1, -1, 1)
# after turning the player 180 degrees to face south.
FLIP = np.diag([1.0, -1.0, -1.0])


class Camera:
    """A three.js PerspectiveCamera placed with lookAt, as viewer.js places it."""

    def __init__(self, view, size, aspect, scale):
        yaw, pitch = np.radians(view["yaw"]), np.radians(view["pitch"])
        target = np.array(view["target"], float)
        d = view["distance"]
        self.position = target + d * np.array([np.cos(pitch) * np.sin(yaw), np.sin(pitch), np.cos(pitch) * np.cos(yaw)])
        z = self.position - target
        z /= np.linalg.norm(z)
        x = np.cross([0, 1, 0], z)
        x /= np.linalg.norm(x)
        y = np.cross(z, x)
        self.rotation = np.stack([x, y, z])  # world to camera
        self.f = 1 / np.tan(np.radians(view["fov"]) / 2)
        self.aspect = aspect
        self.width, self.height = size[0] // scale, size[1] // scale

    def project(self, world):
        c = (world - self.position) @ self.rotation.T
        depth = -c[:, 2]
        nx = self.f / self.aspect * c[:, 0] / depth
        ny = self.f * c[:, 1] / depth
        return np.stack([(nx + 1) / 2 * self.width, (1 - ny) / 2 * self.height], axis=1)


def to_world(points, calib):
    """calib = (scale, tx, ty, tz): where the viewer puts the model."""
    return calib[0] * points @ FLIP.T + np.asarray(calib[1:4])


def splat(px, width, height):
    """A mask covering every projected point, 2x2 pixels each so the surface has no holes."""
    mask = np.zeros((height, width), bool)
    x = np.floor(px[:, 0] - 0.5).astype(int)
    y = np.floor(px[:, 1] - 0.5).astype(int)
    for dx in (0, 1):
        for dy in (0, 1):
            xs, ys = x + dx, y + dy
            ok = (xs >= 0) & (xs < width) & (ys >= 0) & (ys < height)
            mask[ys[ok], xs[ok]] = True
    return mask


def color_features(rgb):
    """Lighting-tolerant colour: hue and saturation as opponent channels divided by brightness, plus
    log brightness. Shading scales all three channels, which leaves the first two alone."""
    lum = rgb.mean(axis=-1)
    o1 = rgb[..., 0] - rgb[..., 1]
    o2 = (rgb[..., 0] + rgb[..., 1]) / 2 - rgb[..., 2]
    return np.stack([o1 / (lum + 0.05), o2 / (lum + 0.05), np.log(lum + 0.05)], axis=-1)


class Target:
    """One captured view of one tick, downscaled, with the distance maps the score needs."""

    def __init__(self, mask, rgb=None):
        self.mask = mask
        self.outside = distance_transform_edt(~mask)
        self.ys, self.xs = np.nonzero(mask)
        self.features = None if rgb is None else color_features(rgb)


def downscale(alpha, scale):
    h, w = alpha.shape
    blocks = alpha[: h // scale * scale, : w // scale * scale].reshape(h // scale, scale, w // scale, scale)
    return blocks.mean(axis=(1, 3)) > 127


def downscale_rgb(rgba, scale):
    """Mean colour of the covered pixels in each block (0-1)."""
    h, w = rgba.shape[:2]
    blocks = rgba[: h // scale * scale, : w // scale * scale].astype(float)
    blocks = blocks.reshape(h // scale, scale, w // scale, scale, 4)
    alpha = blocks[..., 3:] / 255
    return (blocks[..., :3] / 255 * alpha).sum(axis=(1, 3)) / np.maximum(alpha.sum(axis=(1, 3)), 1e-6)


class Capture:
    """The frames and cameras of one capture."""

    def __init__(self, directory, scale=4, views=None):
        self.dir = Path(directory)
        self.meta = json.loads((self.dir / "views.json").read_text())
        self.scale = scale
        self.names = views or list(self.meta["views"])
        self.cameras = {n: Camera(self.meta["views"][n], self.meta["size"], self.meta["aspect"], scale) for n in self.names}

    def targets(self, tick):
        out = {}
        for n in self.names:
            rgba = np.asarray(Image.open(self.dir / n / f"{tick:04d}.png"))
            out[n] = Target(downscale(rgba[:, :, 3], self.scale), downscale_rgb(rgba, self.scale))
        return out


# How much a unit of colour difference counts against a pixel of outline distance, and how much
# brightness counts next to hue and saturation.
COLOR_WEIGHT = 0.5
BRIGHTNESS_WEIGHT = 0.3


def score(points, calib, cameras, targets, per_view=False, colors=None, color_views=None, one_way=False,
          color_weight=None):
    """Mean outline distance in pixels, both ways (our points outside theirs and theirs outside ours),
    plus, with colours, how different the colours are where both outlines cover a pixel.

    one_way only counts our points outside theirs, for fitting some of the parts: the rest of their
    outline is still unexplained, and shouldn't pull the parts being fitted towards it."""
    world = to_world(points, calib)
    total = {}
    for name, cam in cameras.items():
        t = targets[name]
        px = cam.project(world)
        xi = np.clip(px[:, 0].astype(int), 0, cam.width - 1)
        yi = np.clip(px[:, 1].astype(int), 0, cam.height - 1)
        ours_out = t.outside[yi, xi].mean()
        if colors is not None and t.features is not None and (color_views is None or name in color_views):
            depth = -((world - cam.position) @ cam.rotation.T)[:, 2]
            image, mask = splat_color(px, depth, colors, cam.width, cam.height)
            both = mask & t.mask
            diff = np.abs(color_features(image[both]) - t.features[both])
            color = (diff[:, 0] + diff[:, 1] + BRIGHTNESS_WEIGHT * diff[:, 2]).mean() if both.any() else 0
        else:
            mask = splat(px, cam.width, cam.height)
            color = 0
        theirs_out = 0 if one_way or not len(t.ys) else distance_transform_edt(~mask)[t.ys, t.xs].mean()
        total[name] = ours_out + theirs_out + (COLOR_WEIGHT if color_weight is None else color_weight) * color
    return total if per_view else sum(total.values()) / len(total)


def render_masks(points, calib, cameras):
    world = to_world(points, calib)
    return {n: splat(c.project(world), c.width, c.height) for n, c in cameras.items()}


def splat_color(px, depth, colors, width, height):
    """Colour image of the nearest point in each pixel (2x2 pixels per point), and its coverage mask."""
    order = np.argsort(-depth)  # far to near, so nearer points are written last and win
    x = np.floor(px[order, 0] - 0.5).astype(int)
    y = np.floor(px[order, 1] - 0.5).astype(int)
    # All four pixels of every point in one write; a write per offset would let far points overwrite
    # pixels near points filled at another offset.
    xs = np.stack([x, x + 1, x, x + 1], axis=1).ravel()
    ys = np.stack([y, y, y + 1, y + 1], axis=1).ravel()
    cs = np.repeat(colors[order], 4, axis=0)
    ok = (xs >= 0) & (xs < width) & (ys >= 0) & (ys < height)
    image = np.zeros((height, width, 3))
    mask = np.zeros((height, width), bool)
    image[ys[ok], xs[ok]] = cs[ok]
    mask[ys[ok], xs[ok]] = True
    return image, mask


def render_colors(points, colors, calib, cameras):
    world = to_world(points, calib)
    out = {}
    for n, c in cameras.items():
        depth = -((world - c.position) @ c.rotation.T)[:, 2]
        out[n] = splat_color(c.project(world), depth, colors, c.width, c.height)
    return out