diff --git a/.gitignore b/.gitignore index dcb9cb84a..76b54487c 100644 --- a/.gitignore +++ b/.gitignore @@ -182,4 +182,5 @@ outputs*/ node_modules/ scripts/colmap2nerf.py bash/ -nerfactory/ngp_pl/ \ No newline at end of file +nerfactory/ngp_pl/ +cache/ \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 5661ddd76..962180731 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,75 +1,69 @@ { - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Python: Current File", - "type": "python", - "request": "launch", - "program": "${file}", - "console": "integratedTerminal", - "justMyCode": true - }, - { - "name": "Python: run_train (gpu)", - "type": "python", - "request": "launch", - "program": "scripts/run_train.py", - "console": "integratedTerminal" - }, - { - "name": "Python: run_train (cpu)", - "type": "python", - "request": "launch", - "program": "scripts/run_train.py", - "console": "integratedTerminal", - "args": [ - "machine.num_gpus=0" - ] - }, - { - "name": "Python: run_train ngp (gpu)", - "type": "python", - "request": "launch", - "program": "scripts/run_train.py", - "console": "integratedTerminal", - "args": [ - "--config-name=graph_instant_ngp.yaml" - ] - }, - { - "name": "Python: run_train mipnerf 360 (gpu)", - "type": "python", - "request": "launch", - "program": "scripts/run_train.py", - "console": "integratedTerminal", - "args": [ - "--config-name=graph_mipnerf_360.yaml" - ] - }, - { - "name": "Python: run_train nerfw (gpu)", - "type": "python", - "request": "launch", - "program": "scripts/run_train.py", - "console": "integratedTerminal", - "args": [ - "--config-name=graph_nerfw.yaml", - // "++data.dataset.use_cache=true" - ] - }, - { - "name": "Python: run_train semantic_nerf (gpu)", - "type": "python", - "request": "launch", - "program": "scripts/run_train.py", - "console": "integratedTerminal", - "args": [ - "--config-name=graph_semantic_nerf.yaml", - "++data.dataset.use_cache=true" - ] - }, - ] -} \ No newline at end of file + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Current File", + "type": "python", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "justMyCode": true + }, + { + "name": "Python: run_train (gpu)", + "type": "python", + "request": "launch", + "program": "scripts/run_train.py", + "console": "integratedTerminal" + }, + { + "name": "Python: run_train (cpu)", + "type": "python", + "request": "launch", + "program": "scripts/run_train.py", + "console": "integratedTerminal", + "args": ["machine.num_gpus=0"] + }, + { + "name": "Python: run_train ngp (gpu)", + "type": "python", + "request": "launch", + "program": "scripts/run_train.py", + "console": "integratedTerminal", + "args": ["--config-name=graph_instant_ngp.yaml"] + }, + { + "name": "Python: run_train mipnerf 360 (gpu)", + "type": "python", + "request": "launch", + "program": "scripts/run_train.py", + "console": "integratedTerminal", + "args": ["--config-name=graph_mipnerf_360.yaml"] + }, + { + "name": "Python: run_train nerfw (gpu)", + "type": "python", + "request": "launch", + "program": "scripts/run_train.py", + "console": "integratedTerminal", + "args": [ + "--config-name=graph_nerfw.yaml" + // "++data.use_preprocessing_cache=true" + ] + }, + { + "name": "Python: run_train semantic_nerf (gpu)", + "type": "python", + "request": "launch", + "program": "scripts/run_train.py", + "console": "integratedTerminal", + "args": [ + "--config-name=graph_semantic_nerf.yaml", + "++data.use_preprocessing_cache=true" + ] + } + ] +} diff --git a/configs/graph_default.yaml b/configs/graph_default.yaml index 285414d74..8cb739628 100644 --- a/configs/graph_default.yaml +++ b/configs/graph_default.yaml @@ -48,11 +48,9 @@ hydra: data: # Train dataloading. + use_preprocessing_cache: False dataset_inputs_train: - data_directory: data/blender/lego - dataset_format: blender - downscale_factor: 1 - alpha_color: white + _target_: nerfactory.data.datasets.Dataset image_dataset_train: _target_: nerfactory.data.image_dataset.ImageDataset dataloader_train: @@ -65,10 +63,7 @@ data: num_rays_per_batch: 1024 # Eval dataloading. dataset_inputs_eval: - data_directory: data/blender/lego - dataset_format: blender - downscale_factor: 1 # NOTE: this might be the only value that changes - alpha_color: white + _target_: nerfactory.data.datasets.Dataset image_dataset_eval: _target_: nerfactory.data.image_dataset.ImageDataset dataloader_eval: @@ -95,9 +90,9 @@ graph: density_field_config: # Please see the docstr for DensityGrid on how to set those arguments. _target_: nerfactory.fields.density_fields.density_grid.DensityGrid - center: 0.0 # simply set it as the center of the scene bbox - base_scale: 3.0 # simply set it as the scale of the scene bbox - num_cascades: 1 # if using more than 1 cascade, the `base_scale` can be smaller than scene scale. + center: 0.0 # simply set it as the center of the scene bbox + base_scale: 3.0 # simply set it as the scale of the scene bbox + num_cascades: 1 # if using more than 1 cascade, the `base_scale` can be smaller than scene scale. resolution: 128 update_every_num_iters: 16 diff --git a/configs/graph_instant_ngp.yaml b/configs/graph_instant_ngp.yaml index 91ed03068..0b8f04441 100644 --- a/configs/graph_instant_ngp.yaml +++ b/configs/graph_instant_ngp.yaml @@ -4,9 +4,21 @@ defaults: method_name: instant_ngp data: + # Train dataloading. + dataset_inputs_train: + _target_: nerfactory.data.datasets.Blender + data_directory: data/blender/lego + alpha_color: white + downscale_factor: 1 dataloader_train: pixel_sampler: num_rays_per_batch: 8192 # for training + # Eval dataloading + dataset_inputs_eval: + _target_: nerfactory.data.datasets.Blender + data_directory: data/blender/lego + alpha_color: white + downscale_factor: 1 dataloader_eval: num_rays_per_chunk: 8192 # all rays for evaluation @@ -21,7 +33,16 @@ graph: field_implementation: tcnn # torch, tcnn, ... loss_coefficients: # loss coefficients rgb_loss: 1.0 - + + density_field_config: + # Please see the docstr for DensityGrid on how to set those arguments. + _target_: nerfactory.fields.density_fields.density_grid.DensityGrid + center: 0.0 # simply set it as the center of the scene bbox + base_scale: 3.0 # simply set it as the scale of the scene bbox + num_cascades: 1 # if using more than 1 cascade, the `base_scale` can be smaller than scene scale. + resolution: 128 + update_every_num_iters: 16 + # optimizer options for network optimizers: fields: diff --git a/configs/graph_mipnerf.yaml b/configs/graph_mipnerf.yaml index 2b5d7a214..36bf3d015 100644 --- a/configs/graph_mipnerf.yaml +++ b/configs/graph_mipnerf.yaml @@ -9,6 +9,18 @@ method_name: mipnerf # density = softplus(x) density = softplus(x-1) # rgb = sigmoid(x) rgb = (1 + 2e) / (1 + exp(-x)) -e +data: + # Train dataloading. + dataset_inputs_train: + _target_: nerfactory.data.datasets.Blender + data_directory: data/blender/lego + downscale_factor: 1 + # Eval dataloading + dataset_inputs_eval: + _target_: nerfactory.data.datasets.Blender + data_directory: data/blender/lego + downscale_factor: 1 + graph: _target_: nerfactory.graphs.mipnerf.MipNerfGraph num_coarse_samples: 128 diff --git a/configs/graph_mipnerf_360.yaml b/configs/graph_mipnerf_360.yaml index e4fc82997..767899789 100644 --- a/configs/graph_mipnerf_360.yaml +++ b/configs/graph_mipnerf_360.yaml @@ -8,14 +8,14 @@ method_name: vanilla_nerf data: # Train dataloading. dataset_inputs_train: + _target_: nerfactory.data.datasets.Mipnerf360 data_directory: data/mipnerf_360/garden - dataset_format: mipnerf_360 - downscale_factor: 4 + downscale_factor: 1 # Eval dataloading dataset_inputs_eval: + _target_: nerfactory.data.datasets.Mipnerf360 data_directory: data/mipnerf_360/garden - dataset_format: mipnerf_360 - downscale_factor: 8 + downscale_factor: 1 trainer: steps_per_test: 200 diff --git a/configs/graph_nerfw.yaml b/configs/graph_nerfw.yaml index 7514a5a24..2f412925a 100644 --- a/configs/graph_nerfw.yaml +++ b/configs/graph_nerfw.yaml @@ -7,18 +7,20 @@ method_name: nerfw # the data data: + # Train dataloading. + dataset_inputs_train: + _target_: nerfactory.data.datasets.Friends + data_directory: data/friends/TBBT-big_living_room + downscale_factor: 1 image_dataset_train: _target_: nerfactory.data.image_dataset.PanopticImageDataset + # Eval dataloading + dataset_inputs_eval: + _target_: nerfactory.data.datasets.Friends + data_directory: data/friends/TBBT-big_living_room + downscale_factor: 1 image_dataset_eval: _target_: nerfactory.data.image_dataset.PanopticImageDataset - dataset_inputs_train: - data_directory: data/friends/TBBT-big_living_room - dataset_format: friends - downscale_factor: 1 - dataset_inputs_eval: - data_directory: data/friends/TBBT-big_living_room - dataset_format: friends - downscale_factor: 1 graph: _target_: nerfactory.graphs.nerfw.NerfWGraph diff --git a/configs/graph_semantic_nerf.yaml b/configs/graph_semantic_nerf.yaml index ce4ee04a7..54a5343dd 100644 --- a/configs/graph_semantic_nerf.yaml +++ b/configs/graph_semantic_nerf.yaml @@ -7,18 +7,20 @@ method_name: nerfw # the data data: + # Train dataloading. + dataset_inputs_train: + _target_: nerfactory.data.datasets.Friends + data_directory: data/friends/TBBT-big_living_room + downscale_factor: 1 image_dataset_train: _target_: nerfactory.data.image_dataset.PanopticImageDataset + # Eval dataloading + dataset_inputs_eval: + _target_: nerfactory.data.datasets.Friends + data_directory: data/friends/TBBT-big_living_room + downscale_factor: 1 image_dataset_eval: _target_: nerfactory.data.image_dataset.PanopticImageDataset - dataset_inputs_train: - data_directory: data/friends/TBBT-big_living_room - dataset_format: friends - downscale_factor: 1 - dataset_inputs_eval: - data_directory: data/friends/TBBT-big_living_room - dataset_format: friends - downscale_factor: 1 graph: _target_: nerfactory.graphs.semantic_nerf.SemanticNerfGraph diff --git a/configs/graph_vanilla_nerf.yaml b/configs/graph_vanilla_nerf.yaml index d9ce55f91..f0d3a7a29 100644 --- a/configs/graph_vanilla_nerf.yaml +++ b/configs/graph_vanilla_nerf.yaml @@ -2,3 +2,14 @@ defaults: - graph_default - _self_ +data: + # Train dataloading. + dataset_inputs_train: + _target_: nerfactory.data.datasets.Blender + data_directory: data/blender/lego + downscale_factor: 1 + # Eval dataloading + dataset_inputs_eval: + _target_: nerfactory.data.datasets.Blender + data_directory: data/blender/lego + downscale_factor: 1 diff --git a/docs/reference/api/data.rst b/docs/reference/api/data.rst index 2b07b415c..06ec83bbd 100644 --- a/docs/reference/api/data.rst +++ b/docs/reference/api/data.rst @@ -28,12 +28,5 @@ Structures ---------------- .. automodule:: nerfactory.data.structs - :members: - :show-inheritance: - -Utils ----------------- - -.. automodule:: nerfactory.data.utils :members: :show-inheritance: \ No newline at end of file diff --git a/nerfactory/data/dataloader.py b/nerfactory/data/dataloader.py index 5bf036992..d08a81817 100644 --- a/nerfactory/data/dataloader.py +++ b/nerfactory/data/dataloader.py @@ -31,7 +31,6 @@ from nerfactory.data.image_dataset import ImageDataset from nerfactory.data.image_sampler import ImageSampler from nerfactory.data.pixel_sampler import PixelSampler from nerfactory.data.structs import BaseDataContainer, DatasetInputs -from nerfactory.data.utils import get_dataset_inputs_from_dataset_config from nerfactory.graphs.modules.ray_generator import RayGenerator from nerfactory.utils import profiler from nerfactory.utils.config import DataConfig @@ -52,7 +51,10 @@ def setup_dataset_train(config: DataConfig, device: str) -> Tuple[DatasetInputs, Returns: Tuple[DatasetInputs, "TrainDataloader"]: returns both the dataset input information and associated dataloader """ - dataset_inputs_train = get_dataset_inputs_from_dataset_config(**config.dataset_inputs_train, split="train") + dataset_train = instantiate_from_dict_config(config.dataset_inputs_train) + dataset_inputs_train = dataset_train.get_dataset_inputs( + split="train", use_preprocessing_cache=config.use_preprocessing_cache + ) # ImageDataset image_dataset_train = instantiate_from_dict_config(config.image_dataset_train, **dataset_inputs_train.as_dict()) # ImageSampler @@ -78,7 +80,10 @@ def setup_dataset_eval(config: DataConfig, test_mode: bool, device: str) -> Tupl Tuple[DatasetInputs, "TrainDataloader"]: returns both the dataset input information and associated dataloader """ eval_split = "test" if test_mode else "val" - dataset_inputs_eval = get_dataset_inputs_from_dataset_config(**config.dataset_inputs_eval, split=eval_split) + dataset_eval = instantiate_from_dict_config(config.dataset_inputs_eval) + dataset_inputs_eval = dataset_eval.get_dataset_inputs( + split=eval_split, use_preprocessing_cache=config.use_preprocessing_cache + ) image_dataset_eval = instantiate_from_dict_config(config.image_dataset_eval, **dataset_inputs_eval.as_dict()) dataloader_eval = instantiate_from_dict_config( config.dataloader_eval, diff --git a/nerfactory/data/datasets.py b/nerfactory/data/datasets.py new file mode 100644 index 000000000..63cc126f4 --- /dev/null +++ b/nerfactory/data/datasets.py @@ -0,0 +1,511 @@ +# Copyright 2022 The Plenoptix Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A set of standard datasets.""" + +import dataclasses +import logging +import os +from abc import abstractmethod +from dataclasses import dataclass +from typing import Optional, Union + +import imageio +import numpy as np +import torch + +from nerfactory.data.colmap_utils import ( + read_cameras_binary, + read_images_binary, + read_pointsTD_binary, +) +from nerfactory.data.structs import DatasetInputs, PointCloud, SceneBounds, Semantics +from nerfactory.utils.colors import get_color +from nerfactory.utils.io import ( + get_absolute_path, + load_from_json, + load_from_pkl, + make_dir, + write_to_pkl, +) +from nerfactory.utils.misc import get_hash_str_from_dict + + +@dataclass +class Dataset: + """A dataset.""" + + @abstractmethod + def _generate_dataset_inputs(self, split: str = "train") -> DatasetInputs: + """Returns the dataset inputs for the given split. + + Args: + split: Which dataset split to generate. + + Returns: + DatasetInputs + """ + + def get_dataset_inputs(self, split: str = "train", use_preprocessing_cache: bool = False) -> DatasetInputs: + """Returns the dataset inputs for the given split. + + Args: + split: Which dataset split to generate. + use_preprocessing_cache: Whether to use the cached dataset inputs. Defaults to False. + + Returns: + DatasetInputs + """ + if use_preprocessing_cache: + dataset_inputs = self._get_dataset_inputs_from_cache(split) + else: + dataset_inputs = self._generate_dataset_inputs(split) + return dataset_inputs + + def _get_cache_filename(self, split: str) -> str: + """Creates a cache filename from the dataset inputs arguments. + + Args: + split: Which dataset split to generate filename for. + + Returns: + filename for cache. + """ + dataset_config_hash = get_hash_str_from_dict(dataclasses.asdict(self)) + dataset_config_hash_filename = make_dir( + get_absolute_path(f"cache/dataset_inputs/{dataset_config_hash}-{split}.pkl") + ) + return dataset_config_hash_filename + + def save_dataset_inputs_to_cache(self, split: str): + """Saves the dataset inputs to cache. + + Args: + split: Which dataset split to save. + """ + dataset_inputs = self.get_dataset_inputs(split=split) + dataset_config_hash_filename = self._get_cache_filename(split) + write_to_pkl(dataset_config_hash_filename, dataset_inputs) + + def _get_dataset_inputs_from_cache(self, split: str) -> DatasetInputs: + """Loads the dataset inputs from cache. If the cache does not exist, it will be created. + + Args: + split: Which dataset split to load. + """ + dataset_config_hash_filename = self._get_cache_filename(split) + if os.path.exists(dataset_config_hash_filename): + logging.info("Loading dataset from cache.") + dataset_inputs = load_from_pkl(dataset_config_hash_filename) + else: + logging.info("Cache file not found. Generating and saving dataset to cache.") + dataset_inputs = self._generate_dataset_inputs(split=split) + self.save_dataset_inputs_to_cache(split) + return dataset_inputs + + +@dataclass +class Blender(Dataset): + """Blender Dataset + Some of this code comes from https://github.com/yenchenlin/nerf-pytorch/blob/master/load_blender.py#L37. + + Args: + data_directory: Location of data + alpha_color: Sets transparent regions to specified color, otherwise black. + scale_factor: How much to scale the camera origins by. + downscale_factor: How much to downscale images. Defaults to 1. + """ + + data_directory: str + scale_factor: float = 1.0 + alpha_color: Optional[Union[str, list]] = None + downscale_factor: int = 1 + + def _generate_dataset_inputs(self, split="train"): + if self.alpha_color is not None: + alpha_color_tensor = get_color(self.alpha_color) + else: + alpha_color_tensor = None + + abs_dir = get_absolute_path(self.data_directory) + meta = load_from_json(os.path.join(abs_dir, f"transforms_{split}.json")) + image_filenames = [] + poses = [] + for frame in meta["frames"]: + fname = os.path.join(abs_dir, frame["file_path"].replace("./", "") + ".png") + image_filenames.append(fname) + poses.append(np.array(frame["transform_matrix"])) + poses = np.array(poses).astype(np.float32) + + img_0 = imageio.imread(image_filenames[0]) + image_height, image_width = img_0.shape[:2] + camera_angle_x = float(meta["camera_angle_x"]) + focal_length = 0.5 * image_width / np.tan(0.5 * camera_angle_x) + + cx = image_width / 2.0 + cy = image_height / 2.0 + camera_to_world = torch.from_numpy(poses[:, :3]) # camera to world transform + num_cameras = len(image_filenames) + num_intrinsics_params = 3 + intrinsics = torch.ones((num_cameras, num_intrinsics_params), dtype=torch.float32) + intrinsics *= torch.tensor([cx, cy, focal_length]) + + # in x,y,z order + camera_to_world[..., 3] *= self.scale_factor + scene_bounds = SceneBounds(aabb=torch.tensor([[-1.5, -1.5, -1.5], [1.5, 1.5, 1.5]], dtype=torch.float32)) + + dataset_inputs = DatasetInputs( + image_filenames=image_filenames, + downscale_factor=self.downscale_factor, + alpha_color=alpha_color_tensor, + intrinsics=intrinsics * 1.0 / self.downscale_factor, # downscaling the intrinsics here + camera_to_world=camera_to_world, + scene_bounds=scene_bounds, + ) + + return dataset_inputs + + +@dataclass +class InstantNGP(Dataset): + """Instant NGP Dataset + + Args: + data_directory: Location of data + scale_factor: How much to scale the camera origins by. + downscale_factor: How much to downscale images. Defaults to 1. + scene_scale: How much to scale the scene. Defaults to 0.33 + """ + + data_directory: str + scale_factor: float = 1.0 + downscale_factor: int = 1 + scene_scale: float = 0.33 + + def _generate_dataset_inputs(self, split="train"): + + abs_dir = get_absolute_path(self.data_directory) + + meta = load_from_json(os.path.join(abs_dir, "transforms.json")) + image_filenames = [] + poses = [] + num_skipped_image_filenames = 0 + for frame in meta["frames"]: + fname = os.path.join(abs_dir, frame["file_path"]) + if not os.path.exists(fname): + num_skipped_image_filenames += 1 + else: + image_filenames.append(fname) + poses.append(np.array(frame["transform_matrix"])) + if num_skipped_image_filenames >= 0: + logging.info("Skipping %s files in dataset split %s.", num_skipped_image_filenames, split) + assert ( + len(image_filenames) != 0 + ), """ + No image files found. + You should check the file_paths in the transforms.json file to make sure they are correct. + """ + poses = np.array(poses).astype(np.float32) + poses[:3, 3] *= self.scene_scale + + img_0 = imageio.imread(image_filenames[0]) + image_height, image_width = img_0.shape[:2] + camera_angle_x = float(meta["camera_angle_x"]) + focal_length = 0.5 * image_width / np.tan(0.5 * camera_angle_x) + + cx = image_width / 2.0 + cy = image_height / 2.0 + camera_to_world = torch.from_numpy(poses[:, :3]) # camera to world transform + num_cameras = len(image_filenames) + num_intrinsics_params = 3 + intrinsics = torch.ones((num_cameras, num_intrinsics_params), dtype=torch.float32) + intrinsics *= torch.tensor([cx, cy, focal_length]) + + # in x,y,z order + # assumes that the scene is centered at the origin + aabb_scale = meta["aabb_scale"] + scene_bounds = SceneBounds( + aabb=torch.tensor( + [[-aabb_scale, -aabb_scale, -aabb_scale], [aabb_scale, aabb_scale, aabb_scale]], dtype=torch.float32 + ) + ) + + # TODO(ethan): add alpha background color + dataset_inputs = DatasetInputs( + image_filenames=image_filenames, + downscale_factor=self.downscale_factor, + intrinsics=intrinsics * 1.0 / self.downscale_factor, # downscaling the intrinsics here + camera_to_world=camera_to_world, + scene_bounds=scene_bounds, + ) + + return dataset_inputs + + +@dataclass +class Mipnerf360(Dataset): + """MipNeRF 360 Dataset + + Args: + data_directory: Location of data + downscale_factor: How much to downscale images. Defaults to 1. + val_skip: 1/val_skip images to use for validation. Defaults to 8. + auto_scale: Scale based on pose bounds. Defaults to True. + aabb_scale: Scene scale, Defaults to 1.0. + """ + + data_directory: str + downscale_factor: int = 1 + val_skip: int = 8 + auto_scale: bool = True + aabb_scale = 4 + + @classmethod + def _normalize_orientation(cls, poses: np.ndarray): + """Set the _up_ direction to be in the positive Y direction. + + Args: + poses: Numpy array of poses. + """ + poses_orig = poses.copy() + bottom = np.reshape([0, 0, 0, 1.0], [1, 4]) + center = poses[:, :3, 3].mean(0) + vec2 = poses[:, :3, 2].sum(0) / np.linalg.norm(poses[:, :3, 2].sum(0)) + up = poses[:, :3, 1].sum(0) + vec0 = np.cross(up, vec2) / np.linalg.norm(np.cross(up, vec2)) + vec1 = np.cross(vec2, vec0) / np.linalg.norm(np.cross(vec2, vec0)) + c2w = np.stack([vec0, vec1, vec2, center], -1) # [3, 4] + c2w = np.concatenate([c2w[:3, :4], bottom], -2) # [4, 4] + bottom = np.tile(np.reshape(bottom, [1, 1, 4]), [poses.shape[0], 1, 1]) # [BS, 1, 4] + poses = np.concatenate([poses[:, :3, :4], bottom], -2) # [BS, 4, 4] + poses = np.linalg.inv(c2w) @ poses + poses_orig[:, :3, :4] = poses[:, :3, :4] + return poses_orig + + def _generate_dataset_inputs(self, split="train"): + abs_dir = get_absolute_path(self.data_directory) + image_dir = os.path.join(abs_dir, "images") + if self.downscale_factor > 1: + image_dir += f"_{self.downscale_factor}" + + if not os.path.exists(image_dir): + raise ValueError(f"Image directory {image_dir} doesn't exist") + + valid_formats = [".jpg", ".png"] + image_filenames = [] + for f in os.listdir(image_dir): + ext = os.path.splitext(f)[1] + if ext.lower() not in valid_formats: + continue + image_filenames.append(os.path.join(image_dir, f)) + image_filenames = sorted(image_filenames) + num_images = len(image_filenames) + + poses_data = np.load(os.path.join(abs_dir, "poses_bounds.npy")) + poses = poses_data[:, :-2].reshape([-1, 3, 5]).astype(np.float32) + bounds = poses_data[:, -2:].transpose([1, 0]) + + if num_images != poses.shape[0]: + raise RuntimeError(f"Different number of images ({num_images}), and poses ({poses.shape[0]})") + + idx_test = np.arange(num_images)[:: self.val_skip] + idx_train = np.array([i for i in np.arange(num_images) if i not in idx_test]) + idx = idx_train if split == "train" else idx_test + + image_filenames = np.array(image_filenames)[idx] + poses = poses[idx] + + img_0 = imageio.imread(image_filenames[0]) + image_height, image_width = img_0.shape[:2] + + poses[:, :2, 4] = np.array([image_height, image_width]) + poses[:, 2, 4] = poses[:, 2, 4] * 1.0 / self.downscale_factor + + # Reorder pose to match our convention + poses = np.concatenate([poses[:, :, 1:2], -poses[:, :, 0:1], poses[:, :, 2:]], axis=-1) + + # Center poses and rotate. (Compute up from average of all poses) + poses = self._normalize_orientation(poses) + + # Scale factor used in mipnerf + if self.auto_scale: + scale_factor = 1 / (np.min(bounds) * 0.75) + poses[:, :3, 3] *= scale_factor + bounds *= scale_factor + + # Center poses + poses[:, :3, 3] = poses[:, :3, 3] - np.mean(poses[:, :3, :], axis=0)[:, 3] + + focal_length = poses[0, -1, -1] + + cx = image_width / 2.0 + cy = image_height / 2.0 + camera_to_world = torch.from_numpy(poses[:, :3, :4]) # camera to world transform + num_cameras = len(image_filenames) + num_intrinsics_params = 3 + intrinsics = torch.ones((num_cameras, num_intrinsics_params), dtype=torch.float32) + intrinsics *= torch.tensor([cx, cy, focal_length]) + + aabb = torch.tensor([[-4, -4, -4], [4, 4, 4]], dtype=torch.float32) * self.aabb_scale + scene_bounds = SceneBounds(aabb=aabb) + + dataset_inputs = DatasetInputs( + image_filenames=image_filenames, + downscale_factor=1, + intrinsics=intrinsics, + camera_to_world=camera_to_world, + scene_bounds=scene_bounds, + ) + + return dataset_inputs + + +@dataclass +class Friends(Dataset): + """Friends Dataset + + Args: + data_directory: Location of data + downscale_factor: How much to downscale images. Defaults to 1. + include_semantics: whether or not to include the semantics. Defaults to False. + include_point_cloud: whether or not to include the point cloud. Defaults to False. + """ + + data_directory: str + downscale_factor: int = 1 + include_semantics: bool = True + include_point_cloud: bool = False + + @classmethod + def _get_aabb_and_transform(cls, basedir): + """Returns the aabb and pointcloud transform from the threejs.json file.""" + filename = os.path.join(basedir, "threejs.json") + assert os.path.exists(filename) + data = load_from_json(filename) + + # point cloud transformation + transposed_point_cloud_transform = np.array(data["object"]["children"][0]["matrix"]).reshape(4, 4).T + assert transposed_point_cloud_transform[3, 3] == 1.0 + + # bbox transformation + bbox_transform = np.array(data["object"]["children"][1]["matrix"]).reshape(4, 4).T + w, h, d = data["geometries"][1]["width"], data["geometries"][1]["height"], data["geometries"][1]["depth"] + temp = np.array([w, h, d]) / 2.0 + bbox = np.array([-temp, temp]) + bbox = np.concatenate([bbox, np.ones_like(bbox[:, 0:1])], axis=1) + bbox = (bbox_transform @ bbox.T).T[:, 0:3] + + aabb = bbox # rename to aabb because it's an axis-aligned bounding box + return torch.from_numpy(aabb).float(), torch.from_numpy(transposed_point_cloud_transform).float() + + def _generate_dataset_inputs(self, split="train"): # pylint: disable=too-many-statements + + abs_dir = get_absolute_path(self.data_directory) + + images_data = read_images_binary(os.path.join(abs_dir, "colmap", "images.bin")) + # `image_path` is only the end of the filename, including the extension e.g., `.jpg` + image_paths = sorted(os.listdir(os.path.join(abs_dir, "images"))) + + image_path_to_image_id = {} + image_id_to_image_path = {} + for v in images_data.values(): + image_path_to_image_id[v.name] = v.id + image_id_to_image_path[v.id] = v.name + # TODO: handle the splits differently + image_filenames = [os.path.join(abs_dir, "images", image_path) for image_path in image_paths] + + # -- set the bounding box --- + aabb, transposed_point_cloud_transform = self._get_aabb_and_transform(abs_dir) + scene_bounds_original = SceneBounds(aabb=aabb) + # for shifting and rescale accoding to scene bounds + box_center = scene_bounds_original.get_center() + box_scale_factor = 5.0 / scene_bounds_original.get_diagonal_length() # the target diagonal length + scene_bounds = scene_bounds_original.get_centered_and_scaled_scene_bounds(box_scale_factor) + + # --- intrinsics --- + cameras_data = read_cameras_binary(os.path.join(abs_dir, "colmap", "cameras.bin")) + intrinsics = [] + for image_path in image_paths: + cam = cameras_data[image_path_to_image_id[image_path]] + assert len(cam.params) == 3 + focal_length = cam.params[0] # f (fx and fy) + cx = cam.params[1] # cx + cy = cam.params[2] # cy + intrinsics.append([cx, cy, focal_length]) + intrinsics = torch.tensor(intrinsics).float() + + # --- camera_to_world (extrinsics) --- + camera_to_world = [] + bottom_row = np.array([0, 0, 0, 1.0]).reshape(1, 4) + for image_path in image_paths: + image_data = images_data[image_path_to_image_id[image_path]] + rot = image_data.qvec2rotmat() + trans = image_data.tvec.reshape(3, 1) + c2w = np.concatenate([np.concatenate([rot, trans], 1), bottom_row], 0) + camera_to_world.append(c2w) + camera_to_world = torch.tensor(np.array(camera_to_world)).float() + camera_to_world = torch.inverse(camera_to_world) + camera_to_world[..., 1:3] *= -1 + camera_to_world = transposed_point_cloud_transform @ camera_to_world + camera_to_world = camera_to_world[:, :3] + camera_to_world[..., 3] = (camera_to_world[..., 3] - box_center) * box_scale_factor # center and rescale + + # --- semantics --- + semantics = Semantics() + if self.include_semantics: + thing_filenames = [ + image_filename.replace("/images/", "/segmentations/thing/").replace(".jpg", ".png") + for image_filename in image_filenames + ] + stuff_filenames = [ + image_filename.replace("/images/", "/segmentations/stuff/").replace(".jpg", ".png") + for image_filename in image_filenames + ] + panoptic_classes = load_from_json(os.path.join(abs_dir, "panoptic_classes.json")) + stuff_classes = panoptic_classes["stuff"] + stuff_colors = torch.tensor(panoptic_classes["stuff_colors"], dtype=torch.float32) / 255.0 + thing_classes = panoptic_classes["thing"] + thing_colors = torch.tensor(panoptic_classes["thing_colors"], dtype=torch.float32) / 255.0 + semantics = Semantics( + stuff_classes=stuff_classes, + stuff_colors=stuff_colors, + stuff_filenames=stuff_filenames, + thing_classes=thing_classes, + thing_colors=thing_colors, + thing_filenames=thing_filenames, + ) + + # Possibly include the sparse point cloud from COLMAP in the dataset inputs. + # NOTE(ethan): this will be common across the different splits. + point_cloud = PointCloud() + if self.include_point_cloud: + points_3d = read_pointsTD_binary(os.path.join(abs_dir, "colmap", "points3D.bin")) + xyz = torch.tensor(np.array([p_value.xyz for p_id, p_value in points_3d.items()])).float() + rgb = torch.tensor(np.array([p_value.rgb for p_id, p_value in points_3d.items()])).float() + xyz_h = torch.cat([xyz, torch.ones_like(xyz[..., :1])], -1) + xyz = (xyz_h @ transposed_point_cloud_transform.T)[..., :3] + xyz = (xyz - box_center) * box_scale_factor # center and rescale + point_cloud.xyz = xyz + point_cloud.rgb = rgb + + dataset_inputs = DatasetInputs( + image_filenames=image_filenames, + downscale_factor=self.downscale_factor, + intrinsics=intrinsics / self.downscale_factor, + camera_to_world=camera_to_world, + semantics=semantics, + point_cloud=point_cloud, + scene_bounds=scene_bounds, + ) + return dataset_inputs diff --git a/nerfactory/data/format/__init__.py b/nerfactory/data/format/__init__.py deleted file mode 100644 index 02655775e..000000000 --- a/nerfactory/data/format/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2022 The Plenoptix Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/nerfactory/data/format/blender.py b/nerfactory/data/format/blender.py deleted file mode 100644 index d8221822c..000000000 --- a/nerfactory/data/format/blender.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2022 The Plenoptix Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Code to handle loading blender datasets. -""" - -import os -from typing import Optional - -import imageio -import numpy as np -import torch -from torchtyping import TensorType - -from nerfactory.data.structs import DatasetInputs, SceneBounds -from nerfactory.utils.io import load_from_json - - -def load_blender_data( - basedir: str, - downscale_factor: int = 1, - alpha_color: Optional[TensorType[3]] = None, - split: str = "train", - scale_factor: float = 1.0, -) -> DatasetInputs: - """Processes the a blender dataset directory. - Some of this code comes from https://github.com/yenchenlin/nerf-pytorch/blob/master/load_blender.py#L37. - - Args: - data_directory (str): Location of data - downscale_factor (int, optional): How much to downscale images. Defaults to 1.0. - alpha_color (TensorType[3], optional): Sets transparent regions to specified color, otherwise black. - split (str, optional): Which dataset split to generate. - scale_factor (float): How much to scale the camera origins by. - - Returns: - DatasetInputs - """ - # pylint: disable=unused-argument - # pylint: disable-msg=too-many-locals - meta = load_from_json(os.path.join(basedir, f"transforms_{split}.json")) - image_filenames = [] - poses = [] - for frame in meta["frames"]: - fname = os.path.join(basedir, frame["file_path"].replace("./", "") + ".png") - image_filenames.append(fname) - poses.append(np.array(frame["transform_matrix"])) - poses = np.array(poses).astype(np.float32) - - img_0 = imageio.imread(image_filenames[0]) - image_height, image_width = img_0.shape[:2] - camera_angle_x = float(meta["camera_angle_x"]) - focal_length = 0.5 * image_width / np.tan(0.5 * camera_angle_x) - - cx = image_width / 2.0 - cy = image_height / 2.0 - camera_to_world = torch.from_numpy(poses[:, :3]) # camera to world transform - num_cameras = len(image_filenames) - num_intrinsics_params = 3 - intrinsics = torch.ones((num_cameras, num_intrinsics_params), dtype=torch.float32) - intrinsics *= torch.tensor([cx, cy, focal_length]) - - # in x,y,z order - camera_to_world[..., 3] *= scale_factor - scene_bounds = SceneBounds(aabb=torch.tensor([[-1.5, -1.5, -1.5], [1.5, 1.5, 1.5]], dtype=torch.float32)) - - dataset_inputs = DatasetInputs( - image_filenames=image_filenames, - downscale_factor=downscale_factor, - alpha_color=alpha_color, - intrinsics=intrinsics * 1.0 / downscale_factor, # downscaling the intrinsics here - camera_to_world=camera_to_world, - scene_bounds=scene_bounds, - ) - - return dataset_inputs diff --git a/nerfactory/data/format/friends.py b/nerfactory/data/format/friends.py deleted file mode 100644 index be1389d5c..000000000 --- a/nerfactory/data/format/friends.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright 2022 The Plenoptix Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Code to handle loading friends datasets. -""" - -import os - -import numpy as np -import torch - -from nerfactory.data.colmap_utils import ( - read_cameras_binary, - read_images_binary, - read_pointsTD_binary, -) -from nerfactory.data.structs import DatasetInputs, PointCloud, SceneBounds, Semantics -from nerfactory.utils import profiler -from nerfactory.utils.io import load_from_json - - -def get_aabb_and_transform(basedir): - """Returns the aabb and pointcloud transform from the threejs.json file.""" - filename = os.path.join(basedir, "threejs.json") - assert os.path.exists(filename) - data = load_from_json(filename) - - # point cloud transformation - transposed_point_cloud_transform = np.array(data["object"]["children"][0]["matrix"]).reshape(4, 4).T - assert transposed_point_cloud_transform[3, 3] == 1.0 - - # bbox transformation - bbox_transform = np.array(data["object"]["children"][1]["matrix"]).reshape(4, 4).T - w, h, d = data["geometries"][1]["width"], data["geometries"][1]["height"], data["geometries"][1]["depth"] - temp = np.array([w, h, d]) / 2.0 - bbox = np.array([-temp, temp]) - bbox = np.concatenate([bbox, np.ones_like(bbox[:, 0:1])], axis=1) - bbox = (bbox_transform @ bbox.T).T[:, 0:3] - - aabb = bbox # rename to aabb because it's an axis-aligned bounding box - return torch.from_numpy(aabb).float(), torch.from_numpy(transposed_point_cloud_transform).float() - - -@profiler.time_function -def load_friends_data(basedir, downscale_factor=1, split="train", include_semantics=True, include_point_cloud=False): - """_summary_ - - Args: - basedir (_type_): _description_ - downscale_factor (float, optional): _description_. Defaults to 1.0. - split (str, optional): _description_. Defaults to "train". - include_point_cloud (bool): whether or not to include the point cloud - """ - # pylint: disable=unused-argument - # pylint: disable=too-many-statements - - # --- image filenames ---- - images_data = read_images_binary(os.path.join(basedir, "colmap", "images.bin")) - # `image_path` is only the end of the filename, including the extension e.g., `.jpg` - image_paths = sorted(os.listdir(os.path.join(basedir, "images"))) - - image_path_to_image_id = {} - image_id_to_image_path = {} - for v in images_data.values(): - image_path_to_image_id[v.name] = v.id - image_id_to_image_path[v.id] = v.name - # TODO: handle the splits differently - image_filenames = [os.path.join(basedir, "images", image_path) for image_path in image_paths] - - # -- set the bounding box --- - aabb, transposed_point_cloud_transform = get_aabb_and_transform(basedir) - scene_bounds_original = SceneBounds(aabb=aabb) - # for shifting and rescale accoding to scene bounds - box_center = scene_bounds_original.get_center() - box_scale_factor = 5.0 / scene_bounds_original.get_diagonal_length() # the target diagonal length - scene_bounds = scene_bounds_original.get_centered_and_scaled_scene_bounds(box_scale_factor) - - # --- intrinsics --- - cameras_data = read_cameras_binary(os.path.join(basedir, "colmap", "cameras.bin")) - intrinsics = [] - for image_path in image_paths: - cam = cameras_data[image_path_to_image_id[image_path]] - assert len(cam.params) == 3 - focal_length = cam.params[0] # f (fx and fy) - cx = cam.params[1] # cx - cy = cam.params[2] # cy - intrinsics.append([cx, cy, focal_length]) - intrinsics = torch.tensor(intrinsics).float() - - # --- camera_to_world (extrinsics) --- - camera_to_world = [] - bottom_row = np.array([0, 0, 0, 1.0]).reshape(1, 4) - for image_path in image_paths: - image_data = images_data[image_path_to_image_id[image_path]] - rot = image_data.qvec2rotmat() - trans = image_data.tvec.reshape(3, 1) - c2w = np.concatenate([np.concatenate([rot, trans], 1), bottom_row], 0) - camera_to_world.append(c2w) - camera_to_world = torch.tensor(np.array(camera_to_world)).float() - camera_to_world = torch.inverse(camera_to_world) - camera_to_world[..., 1:3] *= -1 - camera_to_world = transposed_point_cloud_transform @ camera_to_world - camera_to_world = camera_to_world[:, :3] - camera_to_world[..., 3] = (camera_to_world[..., 3] - box_center) * box_scale_factor # center and rescale - - # --- semantics --- - semantics = Semantics() - if include_semantics: - thing_filenames = [ - image_filename.replace("/images/", "/segmentations/thing/").replace(".jpg", ".png") - for image_filename in image_filenames - ] - stuff_filenames = [ - image_filename.replace("/images/", "/segmentations/stuff/").replace(".jpg", ".png") - for image_filename in image_filenames - ] - panoptic_classes = load_from_json(os.path.join(basedir, "panoptic_classes.json")) - stuff_classes = panoptic_classes["stuff"] - stuff_colors = torch.tensor(panoptic_classes["stuff_colors"], dtype=torch.float32) / 255.0 - thing_classes = panoptic_classes["thing"] - thing_colors = torch.tensor(panoptic_classes["thing_colors"], dtype=torch.float32) / 255.0 - semantics = Semantics( - stuff_classes=stuff_classes, - stuff_colors=stuff_colors, - stuff_filenames=stuff_filenames, - thing_classes=thing_classes, - thing_colors=thing_colors, - thing_filenames=thing_filenames, - ) - - # Possibly include the sparse point cloud from COLMAP in the dataset inputs. - # NOTE(ethan): this will be common across the different splits. - point_cloud = PointCloud() - if include_point_cloud: - points_3d = read_pointsTD_binary(os.path.join(basedir, "colmap", "points3D.bin")) - xyz = torch.tensor(np.array([p_value.xyz for p_id, p_value in points_3d.items()])).float() - rgb = torch.tensor(np.array([p_value.rgb for p_id, p_value in points_3d.items()])).float() - xyz_h = torch.cat([xyz, torch.ones_like(xyz[..., :1])], -1) - xyz = (xyz_h @ transposed_point_cloud_transform.T)[..., :3] - xyz = (xyz - box_center) * box_scale_factor # center and rescale - point_cloud.xyz = xyz - point_cloud.rgb = rgb - - dataset_inputs = DatasetInputs( - image_filenames=image_filenames, - downscale_factor=downscale_factor, - intrinsics=intrinsics / downscale_factor, - camera_to_world=camera_to_world, - semantics=semantics, - point_cloud=point_cloud, - scene_bounds=scene_bounds, - ) - return dataset_inputs diff --git a/nerfactory/data/format/instant_ngp.py b/nerfactory/data/format/instant_ngp.py deleted file mode 100644 index b08a73655..000000000 --- a/nerfactory/data/format/instant_ngp.py +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright 2022 The Plenoptix Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Code for loading instant-ngp dataset formats. -# TODO -""" - -import logging -import os - -import imageio -import numpy as np -import torch - -from nerfactory.data.structs import DatasetInputs, SceneBounds -from nerfactory.utils.io import load_from_json - - -def load_instant_ngp_data( - basedir: str, downscale_factor: int = 1, split: str = "train", camera_translation_scalar=0.33 -) -> DatasetInputs: - """Processes the a blender dataset directory. - Some of this code comes from https://github.com/yenchenlin/nerf-pytorch/blob/master/load_blender.py#L37. - - Args: - data_directory (str): Location of data - downscale_factor (int, optional): How much to downscale images. Defaults to 1.0. - split (str, optional): Which dataset split to generate. - - Returns: - DatasetInputs - """ - # pylint: disable=unused-argument - # pylint: disable-msg=too-many-locals - meta = load_from_json(os.path.join(basedir, "transforms.json")) - image_filenames = [] - poses = [] - num_skipped_image_filenames = 0 - for frame in meta["frames"]: - fname = os.path.join(basedir, frame["file_path"]) - if not os.path.exists(fname): - num_skipped_image_filenames += 1 - else: - image_filenames.append(fname) - poses.append(np.array(frame["transform_matrix"])) - if num_skipped_image_filenames >= 0: - logging.info("Skipping %s files in dataset split %s.", num_skipped_image_filenames, split) - assert ( - len(image_filenames) != 0 - ), """ - No image files found. - You should check the file_paths in the transforms.json file to make sure they are correct. - """ - poses = np.array(poses).astype(np.float32) - poses[:3, 3] *= camera_translation_scalar - - img_0 = imageio.imread(image_filenames[0]) - image_height, image_width = img_0.shape[:2] - camera_angle_x = float(meta["camera_angle_x"]) - focal_length = 0.5 * image_width / np.tan(0.5 * camera_angle_x) - - cx = image_width / 2.0 - cy = image_height / 2.0 - camera_to_world = torch.from_numpy(poses[:, :3]) # camera to world transform - num_cameras = len(image_filenames) - num_intrinsics_params = 3 - intrinsics = torch.ones((num_cameras, num_intrinsics_params), dtype=torch.float32) - intrinsics *= torch.tensor([cx, cy, focal_length]) - - # in x,y,z order - # assumes that the scene is centered at the origin - aabb_scale = meta["aabb_scale"] - scene_bounds = SceneBounds( - aabb=torch.tensor( - [[-aabb_scale, -aabb_scale, -aabb_scale], [aabb_scale, aabb_scale, aabb_scale]], dtype=torch.float32 - ) - ) - - # TODO(ethan): add alpha background color - dataset_inputs = DatasetInputs( - image_filenames=image_filenames, - downscale_factor=downscale_factor, - intrinsics=intrinsics * 1.0 / downscale_factor, # downscaling the intrinsics here - camera_to_world=camera_to_world, - scene_bounds=scene_bounds, - ) - - return dataset_inputs diff --git a/nerfactory/data/format/llff.py b/nerfactory/data/format/llff.py deleted file mode 100644 index cc52181b7..000000000 --- a/nerfactory/data/format/llff.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2022 The Plenoptix Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Code for loading the LLFF data. -""" - - -def load_llff_data(basedir, downscale_factor=1, split="train", include_point_cloud=False): - """_summary_ - - Args: - basedir (_type_): _description_ - downscale_factor (float, optional): _description_. Defaults to 1.0. - split (str, optional): _description_. Defaults to "train". - include_point_cloud (bool): whether or not to include the point cloud - """ - # pylint: disable=unused-argument - raise NotImplementedError diff --git a/nerfactory/data/format/mipnerf_360.py b/nerfactory/data/format/mipnerf_360.py deleted file mode 100644 index ea44eb2b9..000000000 --- a/nerfactory/data/format/mipnerf_360.py +++ /dev/null @@ -1,139 +0,0 @@ -# Copyright 2022 The Plenoptix Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Code for loading mipnerf 360 dataset formats. -""" - -import os - -import imageio -import numpy as np -import torch - -from nerfactory.data.structs import DatasetInputs, SceneBounds - - -# pylint: disable=too-many-statements -def load_mipnerf_360_data( - basedir: str, - downscale_factor: int = 1, - split: str = "train", - val_skip: int = 8, - auto_scale: bool = True, -) -> DatasetInputs: - """Processes mipnerf 360 data. - Example data can be downloaded from https://jonbarron.info/mipnerf360/. - - TODO: code currently assumes images were previously downscaled. - - Args: - data_directory (str): Location of data - downscale_factor (int, optional): How much to downscale images. Must be a factor of 2. Defaults to 1. - split (str, optional): Which dataset split to generate. - val_skip (int, optional): 1/val_skip images to use for validation. Defaults to 8. - auto_scale (boolean, optional): Scale based on pose bounds. Defaults to True. - aabb_scale (float, optional): Scene scale, Defaults to 1.0. - - Returns: - DatasetInputs - """ - - image_dir = os.path.join(basedir, "images") - if downscale_factor > 1: - image_dir += f"_{downscale_factor}" - - if not os.path.exists(image_dir): - raise ValueError(f"Image directory {image_dir} doesn't exist") - - valid_formats = [".jpg", ".png"] - image_filenames = [] - for f in os.listdir(image_dir): - ext = os.path.splitext(f)[1] - if ext.lower() not in valid_formats: - continue - image_filenames.append(os.path.join(image_dir, f)) - image_filenames = sorted(image_filenames) - num_images = len(image_filenames) - - poses_data = np.load(os.path.join(basedir, "poses_bounds.npy")) - poses = poses_data[:, :-2].reshape([-1, 3, 5]).astype(np.float32) - bounds = poses_data[:, -2:].transpose([1, 0]) - - if num_images != poses.shape[0]: - raise RuntimeError(f"Different number of images ({num_images}), and poses ({poses.shape[0]})") - - idx_test = np.arange(num_images)[::val_skip] - idx_train = np.array([i for i in np.arange(num_images) if i not in idx_test]) - idx = idx_train if split == "train" else idx_test - - image_filenames = np.array(image_filenames)[idx] - poses = poses[idx] - - img_0 = imageio.imread(image_filenames[0]) - image_height, image_width = img_0.shape[:2] - - poses[:, :2, 4] = np.array([image_height, image_width]) - poses[:, 2, 4] = poses[:, 2, 4] * 1.0 / downscale_factor - - # Reorder pose to match our convention - poses = np.concatenate([poses[:, :, 1:2], -poses[:, :, 0:1], poses[:, :, 2:]], axis=-1) - - # Center poses and rotate. (Compute up from average of all poses) - poses_orig = poses.copy() - bottom = np.reshape([0, 0, 0, 1.0], [1, 4]) - center = poses[:, :3, 3].mean(0) - vec2 = poses[:, :3, 2].sum(0) / np.linalg.norm(poses[:, :3, 2].sum(0)) - up = poses[:, :3, 1].sum(0) - vec0 = np.cross(up, vec2) / np.linalg.norm(np.cross(up, vec2)) - vec1 = np.cross(vec2, vec0) / np.linalg.norm(np.cross(vec2, vec0)) - c2w = np.stack([vec0, vec1, vec2, center], -1) # [3, 4] - c2w = np.concatenate([c2w[:3, :4], bottom], -2) # [4, 4] - bottom = np.tile(np.reshape(bottom, [1, 1, 4]), [poses.shape[0], 1, 1]) # [BS, 1, 4] - poses = np.concatenate([poses[:, :3, :4], bottom], -2) # [BS, 4, 4] - poses = np.linalg.inv(c2w) @ poses - poses_orig[:, :3, :4] = poses[:, :3, :4] - poses = poses_orig - - # Scale factor used in mipnerf - if auto_scale: - scale_factor = 1 / (np.min(bounds) * 0.75) - poses[:, :3, 3] *= scale_factor - bounds *= scale_factor - - # Center poses - poses[:, :3, 3] = poses[:, :3, 3] - np.mean(poses[:, :3, :], axis=0)[:, 3] - - focal_length = poses[0, -1, -1] - - cx = image_width / 2.0 - cy = image_height / 2.0 - camera_to_world = torch.from_numpy(poses[:, :3, :4]) # camera to world transform - num_cameras = len(image_filenames) - num_intrinsics_params = 3 - intrinsics = torch.ones((num_cameras, num_intrinsics_params), dtype=torch.float32) - intrinsics *= torch.tensor([cx, cy, focal_length]) - - # TODO: Bounds should be set from config - scene_bounds = SceneBounds(aabb=torch.tensor([[-4, -4, -4], [4, 4, 4]], dtype=torch.float32)) - - dataset_inputs = DatasetInputs( - image_filenames=image_filenames, - downscale_factor=1, - intrinsics=intrinsics, - camera_to_world=camera_to_world, - scene_bounds=scene_bounds, - ) - - return dataset_inputs diff --git a/nerfactory/data/utils.py b/nerfactory/data/utils.py deleted file mode 100644 index 72c112521..000000000 --- a/nerfactory/data/utils.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2022 The Plenoptix Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Code for loading the dataset inputs and for caching the data for fast loading. -""" - -import logging -from typing import Optional, Union - -from omegaconf import ListConfig - -from nerfactory.data.format.blender import load_blender_data -from nerfactory.data.format.friends import load_friends_data -from nerfactory.data.format.instant_ngp import load_instant_ngp_data -from nerfactory.data.format.mipnerf_360 import load_mipnerf_360_data -from nerfactory.data.structs import DatasetInputs -from nerfactory.utils.colors import get_color -from nerfactory.utils.io import get_absolute_path, load_from_pkl, make_dir, write_to_pkl -from nerfactory.utils.misc import get_hash_str_from_dict - - -def get_cache_filename_from_kwargs(kwargs: dict, split: str): - """Creates a cache filename from the dataset inputs arguments.""" - dataset_config_hash = get_hash_str_from_dict(kwargs) - dataset_config_hash_filename = make_dir( - get_absolute_path(f"cache/dataset_inputs/{dataset_config_hash}-{split}.pkl") - ) - return dataset_config_hash_filename - - -def save_dataset_inputs_kwargs_to_cache(kwargs: dict, split: str): - """Saves the dataset inputs to cache.""" - dataset_inputs = get_dataset_inputs(**kwargs, split=split) - dataset_config_hash_filename = get_cache_filename_from_kwargs(kwargs, split) - write_to_pkl(dataset_config_hash_filename, dataset_inputs) - - -def get_dataset_inputs_from_cache(kwargs: dict, split: str): - """Loads the dataset inputs from cache.""" - dataset_config_hash_filename = get_cache_filename_from_kwargs(kwargs, split) - return load_from_pkl(dataset_config_hash_filename) - - -def get_dataset_inputs( - data_directory: str, - dataset_format: str, - split: str, - downscale_factor: int = 1, - alpha_color: Optional[Union[str, list, ListConfig]] = None, -) -> DatasetInputs: - """Returns the dataset inputs, which will be used with an ImageDataset and RayGenerator. - - Args: - data_directory (str): Location of data - dataset_format (str): Name of dataset type - downscale_factor (int, optional): How much to downscale images. Defaults to 1.0. - alpha_color (str, list, optional): Sets transparent regions to specified color, otherwise black. - load_from_cache (bool) - Returns: - DatasetInputs: The inputs needed for generating rays. - """ - - if alpha_color is not None: - alpha_color = get_color(alpha_color) - - if dataset_format == "blender": - dataset_inputs = load_blender_data( - get_absolute_path(data_directory), - downscale_factor=downscale_factor, - split=split, - alpha_color=alpha_color, - ) - elif dataset_format == "friends": - # TODO(ethan): change this hack, and do proper splits for the friends dataset - # currently we assume that there is only the training set of images - dataset_inputs = load_friends_data( - get_absolute_path(data_directory), - downscale_factor=downscale_factor, - split="train", - include_point_cloud=False, - ) - elif dataset_format == "instant_ngp": - dataset_inputs = load_instant_ngp_data( - get_absolute_path(data_directory), - downscale_factor=downscale_factor, - split="train", - ) - elif dataset_format == "mipnerf_360": - dataset_inputs = load_mipnerf_360_data( - get_absolute_path(data_directory), - downscale_factor=downscale_factor, - split="train", - ) - else: - raise NotImplementedError(f"{dataset_format} is not a valid dataset type") - - return dataset_inputs - - -def get_dataset_inputs_from_dataset_config(*, split: str, use_cache: bool = False, **kwargs): - """This may optionally use the cache to return the dataset inputs.""" - if use_cache: - logging.info("Loading from cache! Be careful with using this when making changes!") - print(kwargs) - return get_dataset_inputs_from_cache(kwargs, split) - return get_dataset_inputs(**kwargs, split=split) diff --git a/nerfactory/utils/config.py b/nerfactory/utils/config.py index aa3c06a64..d7d2168a1 100644 --- a/nerfactory/utils/config.py +++ b/nerfactory/utils/config.py @@ -61,16 +61,6 @@ class TrainerConfig: resume_train: ResumeTrainConfig = MISSING -@dataclass -class DatasetInputsConfig: - """Configuration for dataset specification""" - - data_directory: str = MISSING - dataset_format: str = MISSING - downscale_factor: int = MISSING - alpha_color: str = MISSING - - @dataclass class ImageDatasetConfig: """Configuration for type of dataset to instantiate""" @@ -82,14 +72,15 @@ class ImageDatasetConfig: class DataConfig: """Configuration for train/eval datasets""" - dataset_inputs_train: DatasetInputsConfig = MISSING + dataset_inputs_train: Dict[str, Any] = MISSING image_dataset_train: ImageDatasetConfig = MISSING dataloader_train: Dict[str, Any] = MISSING - dataset_inputs_eval: DatasetInputsConfig = MISSING + dataset_inputs_eval: Dict[str, Any] = MISSING image_dataset_eval: ImageDatasetConfig = MISSING dataloader_eval: Dict[str, Any] = MISSING # additional optional parameters here pixel_sampler: Optional[Dict[str, Any]] = None + use_preprocessing_cache: bool = False @dataclass diff --git a/scripts/run_data_preprocessor.py b/scripts/run_data_preprocessor.py deleted file mode 100644 index 74cf05de1..000000000 --- a/scripts/run_data_preprocessor.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -Script to process different dataset types into our format. -""" - -# TODO(ethan): we should write a script here to convert from other datasets to our formats -# e.g., Blender, LLFF, ShapeNet, DTU -from typing import List - -from hydra import compose, initialize - -from nerfactory.data.utils import get_dataset_inputs -from nerfactory.utils.io import get_absolute_path, make_dir, write_to_pkl -from nerfactory.utils.misc import get_hash_str_from_dict - - -def save_dataset_inputs_to_cache( - dataset_name: str = "friends_TBBT-big_living_room", splits: List = None, downscale_factors: List = None -): - """Save the datasets input to cache. - TODO(ethan): use a serializable representation that's cleaner instead of pickle. - - Args: - dataset_name (str): name of the dataset - splits (List[str]): list of splits to save. Defaults to ["train", "val", "test"] - downscale_factors (List[int]): list of downscale factors to save. Defaults to [1, 2, 4] - """ - if splits is None: - splits = ["train", "val", "test"] - if downscale_factors is None: - downscale_factors = [1, 2, 4] - with initialize(version_base="1.2", config_path="../configs"): - config = compose(config_name="default_setup.yaml", overrides=[f"data/dataset={dataset_name}"]) - for downscale_factor in downscale_factors: - for split in splits: - config.data.dataset.downscale_factor = downscale_factor - dataset_config_hash = get_hash_str_from_dict(dict(config.data.dataset)) - dataset_config_hash_filename = make_dir( - get_absolute_path(f"cache/dataset_inputs/{dataset_config_hash}-{split}.pkl") - ) - dataset_inputs = get_dataset_inputs(**config.data.dataset, split=split) - write_to_pkl(dataset_config_hash_filename, dataset_inputs) - - -if __name__ == "__main__": - save_dataset_inputs_to_cache() diff --git a/tests/test_run_train.py b/tests/test_run_train.py index 3ea5c956f..1542027ad 100644 --- a/tests/test_run_train.py +++ b/tests/test_run_train.py @@ -1,3 +1,4 @@ +# pylint: disable=protected-access """ Default test to make sure train runs """ @@ -19,20 +20,22 @@ def set_reduced_config(config: DictConfig): config.trainer.max_num_iterations = 2 # reduce dataset factors; set dataset to test # switch to using the vanilla ImageDataset class - config.data.image_dataset_train._target_ = ( # pylint: disable=protected-access - "nerfactory.data.image_dataset.ImageDataset" - ) - config.data.dataset_inputs_train.data_directory = "tests/data/lego_test" - config.data.dataset_inputs_train.dataset_format = "blender" - config.data.dataset_inputs_train.downscale_factor = 16 + config.data.image_dataset_train._target_ = "nerfactory.data.image_dataset.ImageDataset" + + config.data.dataset_inputs_train = { + "_target_": "nerfactory.data.datasets.Blender", + "data_directory": "tests/data/lego_test", + "downscale_factor": 16, + } config.data.dataloader_train.image_sampler.num_images_to_sample_from = 1 config.data.dataloader_train.pixel_sampler.num_rays_per_batch = 4 - config.data.image_dataset_eval._target_ = ( # pylint: disable=protected-access - "nerfactory.data.image_dataset.ImageDataset" - ) - config.data.dataset_inputs_eval.data_directory = "tests/data/lego_test" - config.data.dataset_inputs_eval.dataset_format = "blender" - config.data.dataset_inputs_eval.downscale_factor = 16 + config.data.image_dataset_eval._target_ = "nerfactory.data.image_dataset.ImageDataset" + config.data.dataset_inputs_eval = { + "_target_": "nerfactory.data.datasets.Blender", + "data_directory": "tests/data/lego_test", + "downscale_factor": 16, + } + # reduce graph factors config.graph.num_coarse_samples = 4 config.graph.num_importance_samples = 4