# Copyright 2022 The Plenoptix Team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Data parser for friends dataset""" from __future__ import annotations from dataclasses import dataclass from pathlib import Path import numpy as np import torch from PIL import Image from rich.console import Console import nerfactory.configs.base as cfg from nerfactory.cameras.cameras import Cameras, CameraType from nerfactory.datamanagers.dataparsers.base import DataParser from nerfactory.datamanagers.structs import DatasetInputs, SceneBounds, Semantics from nerfactory.utils.io import get_absolute_path, load_from_json CONSOLE = Console() def get_semantics_and_masks(image_idx: int, semantics: Semantics): """function to process additional semantics and mask information Args: image_idx: specific image index to work with semantics: semantics data """ # handle mask person_index = semantics.thing_classes.index("person") thing_image_filename = semantics.thing_filenames[image_idx] pil_image = Image.open(thing_image_filename) thing_semantics = torch.from_numpy(np.array(pil_image, dtype="int32"))[..., None] mask = (thing_semantics != person_index).to(torch.float32) # 1 where valid # handle semantics stuff_image_filename = semantics.stuff_filenames[image_idx] pil_image = Image.open(stuff_image_filename) stuff_semantics = torch.from_numpy(np.array(pil_image, dtype="int32"))[..., None] return {"mask": mask, "semantics": stuff_semantics} @dataclass class Friends(DataParser): """Friends Dataset""" config: cfg.FriendsDataParserConfig def _generate_dataset_inputs(self, split="train"): # pylint: disable=unused-argument abs_dir = get_absolute_path(self.config.data_directory) cameras_json = load_from_json(abs_dir / "cameras.json") frames = cameras_json["frames"] bbox = torch.tensor(cameras_json["bbox"]) downscale_suffix = f"_{self.config.downscale_factor}" if self.config.downscale_factor != 1 else "" images_folder = f"images{downscale_suffix}" segmentations_folder = f"segmentations{downscale_suffix}" image_filenames = [] fx = [] fy = [] cx = [] cy = [] camera_to_worlds = [] for frame in frames: # unpack data image_filename = abs_dir / images_folder / frame["image_name"] intrinsics = torch.tensor(frame["intrinsics"]) camtoworld = torch.tensor(frame["camtoworld"])[:3] # append data image_filenames.append(image_filename) fx.append(intrinsics[0, 0]) fy.append(intrinsics[1, 1]) cx.append(intrinsics[0, 2]) cy.append(intrinsics[1, 2]) camera_to_worlds.append(camtoworld) fx = torch.stack(fx) fy = torch.stack(fy) cx = torch.stack(cx) cy = torch.stack(cy) camera_to_worlds = torch.stack(camera_to_worlds) # rotate the cameras and box 90 degrees about the x axis to put the z axis up rotation = torch.tensor([[1, 0, 0], [0, 0, -1], [0, 1, 0]], dtype=torch.float32) camera_to_worlds[:, :3] = rotation @ camera_to_worlds[:, :3] bbox = (rotation @ bbox.T).T scene_scale = self.config.scene_scale # -- set the scene bounds --- scene_bounds = SceneBounds(aabb=bbox) # center the box and adjust the cameras too center = scene_bounds.get_center() scene_bounds.aabb -= center camera_to_worlds[..., 3] -= center # scale the longest dimension to match the cube size lengths = scene_bounds.aabb[1] - scene_bounds.aabb[0] longest_dim = torch.argmax(lengths) longest_length = lengths[longest_dim] scale = scene_scale / longest_length scene_bounds.aabb = scene_bounds.aabb * scale # box camera_to_worlds[..., 3] *= scale # cameras # --- semantics --- semantics = None if self.config.include_semantics: thing_filenames = [ Path( str(image_filename) .replace(f"/{images_folder}/", f"/{segmentations_folder}/thing/") .replace(".jpg", ".png") ) for image_filename in image_filenames ] stuff_filenames = [ Path( str(image_filename) .replace(f"/{images_folder}/", f"/{segmentations_folder}/stuff/") .replace(".jpg", ".png") ) for image_filename in image_filenames ] panoptic_classes = load_from_json(abs_dir / "panoptic_classes.json") stuff_classes = panoptic_classes["stuff"] stuff_colors = torch.tensor(panoptic_classes["stuff_colors"], dtype=torch.float32) / 255.0 thing_classes = panoptic_classes["thing"] thing_colors = torch.tensor(panoptic_classes["thing_colors"], dtype=torch.float32) / 255.0 semantics = Semantics( stuff_classes=stuff_classes, stuff_colors=stuff_colors, stuff_filenames=stuff_filenames, thing_classes=thing_classes, thing_colors=thing_colors, thing_filenames=thing_filenames, ) assert torch.all(cx[0] == cx), "Not all cameras have the same cx. Our Cameras class does not support this." assert torch.all(cy[0] == cy), "Not all cameras have the same cy. Our Cameras class does not support this." cameras = Cameras( fx=fx, fy=fy, cx=float(cx[0]), cy=float(cy[0]), camera_to_worlds=camera_to_worlds, camera_type=CameraType.PERSPECTIVE, ) cameras.rescale_output_resolution(scaling_factor=1.0 / self.config.downscale_factor) dataset_inputs = DatasetInputs( image_filenames=image_filenames, cameras=cameras, scene_bounds=scene_bounds, additional_inputs={"semantics": {"func": get_semantics_and_masks, "kwargs": {"semantics": semantics}}}, ) return dataset_inputs