nerfstudio/nerfactory/datamanagers/dataparsers/friends_parser.py

# Copyright 2022 The Plenoptix Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Data parser for friends dataset"""
from __future__ import annotations

from dataclasses import dataclass, field
from pathlib import Path
from typing import Type

import numpy as np
import torch
from PIL import Image
from rich.console import Console

from nerfactory.cameras.cameras import Cameras, CameraType
from nerfactory.datamanagers.dataparsers.base import DataParser, DataParserConfig
from nerfactory.datamanagers.structs import DatasetInputs, SceneBounds, Semantics
from nerfactory.utils.io import get_absolute_path, load_from_json

CONSOLE = Console()


def get_semantics_and_masks(image_idx: int, semantics: Semantics):
    """function to process additional semantics and mask information

    Args:
        image_idx: specific image index to work with
        semantics: semantics data
    """
    # handle mask
    person_index = semantics.thing_classes.index("person")
    thing_image_filename = semantics.thing_filenames[image_idx]
    pil_image = Image.open(thing_image_filename)
    thing_semantics = torch.from_numpy(np.array(pil_image, dtype="int32"))[..., None]
    mask = (thing_semantics != person_index).to(torch.float32)  # 1 where valid
    # handle semantics
    stuff_image_filename = semantics.stuff_filenames[image_idx]
    pil_image = Image.open(stuff_image_filename)
    stuff_semantics = torch.from_numpy(np.array(pil_image, dtype="int32"))[..., None]
    return {"mask": mask, "semantics": stuff_semantics}


@dataclass
class FriendsDataParserConfig(DataParserConfig):
    """Friends dataset parser config"""

    _target: Type = field(default_factory=lambda: Friends)
    """target class to instantiate"""
    data_directory: Path = Path("data/friends/TBBT-big_living_room")
    """directory specifying location of data"""
    include_semantics: bool = True
    """whether or not to include loading of semantics data"""
    downscale_factor: int = 8
    scene_scale: float = 4.0
    """
    Sets the bounding cube to have edge length of this size.
    The longest dimension of the Friends axis-aligned bbox will be scaled to this value.
    """


@dataclass
class Friends(DataParser):
    """Friends Dataset"""

    config: FriendsDataParserConfig

    def _generate_dataset_inputs(self, split="train"):  # pylint: disable=unused-argument,too-many-statements

        abs_dir = get_absolute_path(self.config.data_directory)

        cameras_json = load_from_json(abs_dir / "cameras.json")
        frames = cameras_json["frames"]
        bbox = torch.tensor(cameras_json["bbox"])

        downscale_suffix = f"_{self.config.downscale_factor}" if self.config.downscale_factor != 1 else ""
        images_folder = f"images{downscale_suffix}"
        segmentations_folder = f"segmentations{downscale_suffix}"

        image_filenames = []
        fx = []
        fy = []
        cx = []
        cy = []
        camera_to_worlds = []
        for frame in frames:
            # unpack data
            image_filename = abs_dir / images_folder / frame["image_name"]
            intrinsics = torch.tensor(frame["intrinsics"])
            camtoworld = torch.tensor(frame["camtoworld"])[:3]
            # append data
            image_filenames.append(image_filename)
            fx.append(intrinsics[0, 0])
            fy.append(intrinsics[1, 1])
            cx.append(intrinsics[0, 2])
            cy.append(intrinsics[1, 2])
            camera_to_worlds.append(camtoworld)
        fx = torch.stack(fx)
        fy = torch.stack(fy)
        cx = torch.stack(cx)
        cy = torch.stack(cy)
        camera_to_worlds = torch.stack(camera_to_worlds)

        # rotate the cameras and box 90 degrees about the x axis to put the z axis up
        rotation = torch.tensor([[1, 0, 0], [0, 0, -1], [0, 1, 0]], dtype=torch.float32)
        camera_to_worlds[:, :3] = rotation @ camera_to_worlds[:, :3]
        bbox = (rotation @ bbox.T).T

        scene_scale = self.config.scene_scale

        # -- set the scene bounds ---
        scene_bounds = SceneBounds(aabb=bbox)
        # center the box and adjust the cameras too
        center = scene_bounds.get_center()
        scene_bounds.aabb -= center
        camera_to_worlds[..., 3] -= center
        # scale the longest dimension to match the cube size
        lengths = scene_bounds.aabb[1] - scene_bounds.aabb[0]
        longest_dim = torch.argmax(lengths)
        longest_length = lengths[longest_dim]
        scale = scene_scale / longest_length
        scene_bounds.aabb = scene_bounds.aabb * scale  # box
        camera_to_worlds[..., 3] *= scale  # cameras

        # --- semantics ---
        semantics = None
        if self.config.include_semantics:
            thing_filenames = [
                Path(
                    str(image_filename)
                    .replace(f"/{images_folder}/", f"/{segmentations_folder}/thing/")
                    .replace(".jpg", ".png")
                )
                for image_filename in image_filenames
            ]
            stuff_filenames = [
                Path(
                    str(image_filename)
                    .replace(f"/{images_folder}/", f"/{segmentations_folder}/stuff/")
                    .replace(".jpg", ".png")
                )
                for image_filename in image_filenames
            ]
            panoptic_classes = load_from_json(abs_dir / "panoptic_classes.json")
            stuff_classes = panoptic_classes["stuff"]
            stuff_colors = torch.tensor(panoptic_classes["stuff_colors"], dtype=torch.float32) / 255.0
            thing_classes = panoptic_classes["thing"]
            thing_colors = torch.tensor(panoptic_classes["thing_colors"], dtype=torch.float32) / 255.0
            semantics = Semantics(
                stuff_classes=stuff_classes,
                stuff_colors=stuff_colors,
                stuff_filenames=stuff_filenames,
                thing_classes=thing_classes,
                thing_colors=thing_colors,
                thing_filenames=thing_filenames,
            )

        assert torch.all(cx[0] == cx), "Not all cameras have the same cx. Our Cameras class does not support this."
        assert torch.all(cy[0] == cy), "Not all cameras have the same cy. Our Cameras class does not support this."

        cameras = Cameras(
            fx=fx,
            fy=fy,
            cx=float(cx[0]),
            cy=float(cy[0]),
            camera_to_worlds=camera_to_worlds,
            camera_type=CameraType.PERSPECTIVE,
        )
        cameras.rescale_output_resolution(scaling_factor=1.0 / self.config.downscale_factor)

        dataset_inputs = DatasetInputs(
            image_filenames=image_filenames,
            cameras=cameras,
            scene_bounds=scene_bounds,
            additional_inputs={"semantics": {"func": get_semantics_and_masks, "kwargs": {"semantics": semantics}}},
        )
        return dataset_inputs