From 3efd3c3298c68000d8f96972382d6dc552cafd21 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Mon, 3 Oct 2022 09:45:36 -0700 Subject: [PATCH] dataset_inputs -> dataparser_outputs (#627) --- docs/tutorials/data/choosing_dataset.md | 79 ------------------- docs/tutorials/data/custom_dataset_format.md | 46 ----------- docs/tutorials/data/index.rst | 2 - docs/tutorials/pipelines/dataparsers.ipynb | 12 +-- nerfstudio/cameras/cameras.py | 2 +- nerfstudio/data/datamanagers.py | 10 +-- .../data/dataparsers/base_dataparser.py | 20 ++--- .../data/dataparsers/blender_dataparser.py | 6 +- .../data/dataparsers/friends_dataparser.py | 6 +- .../dataparsers/instant_ngp_dataparser.py | 6 +- .../data/dataparsers/mipnerf_dataparser.py | 6 +- .../data/dataparsers/nerfstudio_dataparser.py | 6 +- .../data/dataparsers/record3d_dataparser.py | 6 +- nerfstudio/data/utils/dataloaders.py | 2 +- nerfstudio/data/utils/datasets.py | 16 ++-- nerfstudio/pipelines/base_pipeline.py | 2 +- nerfstudio/viewer/server/viewer_utils.py | 4 +- 17 files changed, 52 insertions(+), 179 deletions(-) delete mode 100644 docs/tutorials/data/choosing_dataset.md delete mode 100644 docs/tutorials/data/custom_dataset_format.md diff --git a/docs/tutorials/data/choosing_dataset.md b/docs/tutorials/data/choosing_dataset.md deleted file mode 100644 index ccb2d6389..000000000 --- a/docs/tutorials/data/choosing_dataset.md +++ /dev/null @@ -1,79 +0,0 @@ -# Dataset configuration - -#### Specifying dataset - -The dataset path and type is specified in the model config. Both `dataset_inputs_train` and a `dataset_inputs_eval` need to be set. You can see `configs/graph_default.yaml` for more details on what a full config would look like. - -```yaml -data: - dataset_inputs_train: - data_directory: data/instant_ngp/fox - dataset_format: instant_ngp - alpha_color: white - dataset_inputs_eval: - data_directory: data/instant_ngp/fox - dataset_format: instant_ngp - alpha_color: white -``` - -#### Visualize dataset - -TODO - -#### Supported datasets - -We include support for a set of standard dataset formats. - -###### Blender Synthetic - -```{button-link} https://drive.google.com/drive/u/1/folders/128yBriW1IG_3NJ5Rp7APSTZsJqdJdfc1 -:color: primary -:outline: -Download -``` - -```yaml -dataset_format: blender -``` - -Classic NeRF dataset including the "lego bulldozer" scene that was released with the original NeRF paper. - -###### Instant NGP - -```{button-link} https://github.com/NVlabs/instant-ngp#nerf-fox -:color: primary -:outline: -Download -``` - -```yaml -dataset_format: instant_ngp -``` - -###### MipNeRF 360 - -```{button-link} https://jonbarron.info/mipnerf360/ -:color: primary -:outline: -Download -``` - -```yaml -dataset_format: mipnerf_360 -``` - -###### Record3D - -```{button-link} https://record3d.app/ -:color: primary -:outline: -Download -``` - -```yaml -dataset_format: record_3d -``` - -Directly import dataset recorded from a >= iPhone 12 Pro using the Record3D app. - -Record a video and export with the `EXR + JPG sequence` format. Unzip export and `rgb` folder before training. diff --git a/docs/tutorials/data/custom_dataset_format.md b/docs/tutorials/data/custom_dataset_format.md deleted file mode 100644 index 271914fbc..000000000 --- a/docs/tutorials/data/custom_dataset_format.md +++ /dev/null @@ -1,46 +0,0 @@ -# Custom dataset formats - -We've implemented many common datasets inside the `nerfstudio/format/.py` files. If you have a posed image dataset that does not match these existing formats can create a custom dataset format. - -1. Create a function `nerfstudio/format/.py` and implements a function called `load__data` returns a `DatasetInputs` object. - -```python -# The DatasetInputs dataclass that is passed around to create datasets and initialize the graphs. -# See `nerfstudio/data/structs.py` for the code. -@dataclass -class DatasetInputs: - """Dataset inputs are used to initialize datasets and the NeRF graph.""" - - image_filenames: List[str] - intrinsics: torch.tensor = None - camera_to_world: torch.tensor = None - mask_filenames: List[str] = None - depth_filenames: List[str] = None - scene_box: SceneBox = SceneBox() - semantics: Semantics = Semantics() - point_cloud: PointCloud = PointCloud() - alpha_color: Optional[TensorType[3]] = None - -# The method signature for the `instant_ngp` dataset format. -# See `nerfstudio/data/format/instant_ngp.py` for the code. -def load_instant_ngp_data( - basedir: str, - split: str = "train", camera_translation_scalar=0.33 -) -> DatasetInputs: - """Returns a DatasetInputs struct.""" -``` - -2. Add a new case to `get_dataset_inputs()` in `nerfstudio/data/utils.py` for the new dataset. - -```python -# The method signature for get_dataset_inputs. -# Notice that the parameters match the yaml config parameters above. -# See `nerfstudio/data/utils.py` for the code. -def get_dataset_inputs( - data_directory: str, - dataset_format: str, - split: str, - alpha_color: Optional[Union[str, list, ListConfig]] = None, -) -> DatasetInputs: - """Makes a call to `load__data` and returns a DatasetInputs struct.""" -``` diff --git a/docs/tutorials/data/index.rst b/docs/tutorials/data/index.rst index 7d1d6c888..fde6bf107 100644 --- a/docs/tutorials/data/index.rst +++ b/docs/tutorials/data/index.rst @@ -4,6 +4,4 @@ Data Setup .. toctree:: :maxdepth: 1 - choosing_dataset.md creating_dataset.md - custom_dataset_format.md \ No newline at end of file diff --git a/docs/tutorials/pipelines/dataparsers.ipynb b/docs/tutorials/pipelines/dataparsers.ipynb index cc92f3fe0..b4bdd85c6 100644 --- a/docs/tutorials/pipelines/dataparsers.ipynb +++ b/docs/tutorials/pipelines/dataparsers.ipynb @@ -17,7 +17,7 @@ "```python\n", "@dataclass\n", "class DataparserOutputs:\n", - " \"\"\"Dataset inputs for the image dataset and the ray generator.\"\"\"\n", + " \"\"\"Dataparser outputs for the image dataset and the ray generator.\"\"\"\n", "\n", " image_filenames: List[Path]\n", " \"\"\"Filenames for the images.\"\"\"\n", @@ -33,7 +33,7 @@ "class DataParser:\n", "\n", " @abstractmethod\n", - " def _generate_dataset_inputs(self, split: str = \"train\") -> DataparserOutputs:\n", + " def _generate_datapaser_outputs(self, split: str = \"train\") -> DataparserOutputs:\n", " \"\"\"Abstract method that returns the dataparser outputs for the given split.\n", "\n", " Args:\n", @@ -75,7 +75,7 @@ "\n", " config: NerfstudioDataParserConfig\n", "\n", - " def _generate_dataset_inputs(self, split=\"train\") -> DataParserOutputs:\n", + " def _generate_datapaser_outputs(self, split=\"train\") -> DataParserOutputs:\n", " \"\"\"Returns DataParserOutputs.\"\"\"\n", "```\n", "\n", @@ -138,18 +138,18 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.8.13 ('nerfactory')", + "display_name": "Python 3.8.12 ('nerfactory')", "language": "python", "name": "python3" }, "language_info": { "name": "python", - "version": "3.8.13" + "version": "3.8.12" }, "orig_nbformat": 4, "vscode": { "interpreter": { - "hash": "b7b86c17c0dfd6b313529972d424f2e075cc198a46cda005757126afe8133c7b" + "hash": "34c28001ff35fb390494047002768a8182dcf55b1b11415165e62ea61557ab83" } } }, diff --git a/nerfstudio/cameras/cameras.py b/nerfstudio/cameras/cameras.py index 103149892..ef26d1ff6 100644 --- a/nerfstudio/cameras/cameras.py +++ b/nerfstudio/cameras/cameras.py @@ -49,7 +49,7 @@ CAMERA_MODEL_TO_TYPE = { class Cameras: - """Dataset inputs for the image dataset and the ray generator. + """Dataparser outputs for the image dataset and the ray generator. Note: currently only supports cameras with the same principal points and types. The reason we type the focal lengths, principal points, and image sizes as tensors is to allow for batched cameras diff --git a/nerfstudio/data/datamanagers.py b/nerfstudio/data/datamanagers.py index b84b536cb..02578a79d 100644 --- a/nerfstudio/data/datamanagers.py +++ b/nerfstudio/data/datamanagers.py @@ -284,9 +284,9 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method self.local_rank = local_rank self.sampler = None - self.train_dataset = InputDataset(config.dataparser.setup().get_dataset_inputs(split="train")) + self.train_dataset = InputDataset(config.dataparser.setup().get_dataparser_outputs(split="train")) self.eval_dataset = InputDataset( - config.dataparser.setup().get_dataset_inputs(split="val" if not test_mode else "test") + config.dataparser.setup().get_dataparser_outputs(split="val" if not test_mode else "test") ) super().__init__() @@ -310,10 +310,10 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method self.iter_train_image_dataloader = iter(self.train_image_dataloader) self.train_pixel_sampler = PixelSampler(self.config.train_num_rays_per_batch) self.train_camera_optimizer = self.config.train_camera_optimizer.setup( - num_cameras=self.train_dataset.dataset_inputs.cameras.size, device=self.device + num_cameras=self.train_dataset.dataparser_outputs.cameras.size, device=self.device ) self.train_ray_generator = RayGenerator( - self.train_dataset.dataset_inputs.cameras.to(self.device), + self.train_dataset.dataparser_outputs.cameras.to(self.device), self.train_camera_optimizer, ) @@ -337,7 +337,7 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method self.iter_eval_image_dataloader = iter(self.eval_image_dataloader) self.eval_pixel_sampler = PixelSampler(self.config.eval_num_rays_per_batch) self.eval_ray_generator = RayGenerator( - self.eval_dataset.dataset_inputs.cameras.to(self.device), + self.eval_dataset.dataparser_outputs.cameras.to(self.device), self.train_camera_optimizer, # should be shared between train and eval. ) # for loading full images diff --git a/nerfstudio/data/dataparsers/base_dataparser.py b/nerfstudio/data/dataparsers/base_dataparser.py index 05dbd5e15..ce083d5f7 100644 --- a/nerfstudio/data/dataparsers/base_dataparser.py +++ b/nerfstudio/data/dataparsers/base_dataparser.py @@ -50,7 +50,7 @@ class Semantics: @dataclass class DataparserOutputs: - """Dataset inputs for the image dataset and the ray generator.""" + """Dataparser outputs for the image dataset and the ray generator.""" image_filenames: List[Path] """Filenames for the images.""" @@ -64,7 +64,7 @@ class DataparserOutputs: additional_inputs: Dict[str, Any] = to_immutable_dict({}) """Dictionary of additional dataset information (e.g. semantics/point clouds/masks). {input_name: - ... {"func": function to process additional dataset inputs, + ... {"func": function to process additional dataparser outputs, ... "kwargs": dictionary of data to pass into "func"} } """ @@ -97,24 +97,24 @@ class DataParser: self.config = config @abstractmethod - def _generate_dataset_inputs(self, split: str = "train") -> DataparserOutputs: - """Abstract method that returns the dataset inputs for the given split. + def _generate_dataparser_outputs(self, split: str = "train") -> DataparserOutputs: + """Abstract method that returns the dataparser outputs for the given split. Args: split: Which dataset split to generate (train/test). Returns: - DatasetInputs containing data for the specified dataset and split + DataparserOutputs containing data for the specified dataset and split """ - def get_dataset_inputs(self, split: str = "train") -> DataparserOutputs: - """Returns the dataset inputs for the given split. + def get_dataparser_outputs(self, split: str = "train") -> DataparserOutputs: + """Returns the dataparser outputs for the given split. Args: split: Which dataset split to generate (train/test). Returns: - DatasetInputs containing data for the specified dataset and split + DataparserOutputs containing data for the specified dataset and split """ - dataset_inputs = self._generate_dataset_inputs(split) - return dataset_inputs + dataparser_outputs = self._generate_dataparser_outputs(split) + return dataparser_outputs diff --git a/nerfstudio/data/dataparsers/blender_dataparser.py b/nerfstudio/data/dataparsers/blender_dataparser.py index 1612e5086..7f6a62a13 100644 --- a/nerfstudio/data/dataparsers/blender_dataparser.py +++ b/nerfstudio/data/dataparsers/blender_dataparser.py @@ -62,7 +62,7 @@ class Blender(DataParser): self.scale_factor: float = config.scale_factor self.alpha_color = config.alpha_color - def _generate_dataset_inputs(self, split="train"): + def _generate_dataparser_outputs(self, split="train"): if self.alpha_color is not None: alpha_color_tensor = get_color(self.alpha_color) else: @@ -99,11 +99,11 @@ class Blender(DataParser): camera_type=CameraType.PERSPECTIVE, ) - dataset_inputs = DataparserOutputs( + dataparser_outputs = DataparserOutputs( image_filenames=image_filenames, cameras=cameras, alpha_color=alpha_color_tensor, scene_box=scene_box, ) - return dataset_inputs + return dataparser_outputs diff --git a/nerfstudio/data/dataparsers/friends_dataparser.py b/nerfstudio/data/dataparsers/friends_dataparser.py index 1f4648712..1926b4afb 100644 --- a/nerfstudio/data/dataparsers/friends_dataparser.py +++ b/nerfstudio/data/dataparsers/friends_dataparser.py @@ -81,7 +81,7 @@ class Friends(DataParser): config: FriendsDataParserConfig - def _generate_dataset_inputs(self, split="train"): # pylint: disable=unused-argument,too-many-statements + def _generate_dataparser_outputs(self, split="train"): # pylint: disable=unused-argument,too-many-statements cameras_json = load_from_json(self.config.data_directory / "cameras.json") frames = cameras_json["frames"] @@ -182,10 +182,10 @@ class Friends(DataParser): ) cameras.rescale_output_resolution(scaling_factor=1.0 / self.config.downscale_factor) - dataset_inputs = DataparserOutputs( + dataparser_outputs = DataparserOutputs( image_filenames=image_filenames, cameras=cameras, scene_box=scene_box, additional_inputs={"semantics": {"func": get_semantics_and_masks, "kwargs": {"semantics": semantics}}}, ) - return dataset_inputs + return dataparser_outputs diff --git a/nerfstudio/data/dataparsers/instant_ngp_dataparser.py b/nerfstudio/data/dataparsers/instant_ngp_dataparser.py index a6f2bf488..a5578aa62 100644 --- a/nerfstudio/data/dataparsers/instant_ngp_dataparser.py +++ b/nerfstudio/data/dataparsers/instant_ngp_dataparser.py @@ -55,7 +55,7 @@ class InstantNGP(DataParser): config: InstantNGPDataParserConfig - def _generate_dataset_inputs(self, split="train"): + def _generate_dataparser_outputs(self, split="train"): meta = load_from_json(self.config.data_directory / "transforms.json") image_filenames = [] @@ -107,10 +107,10 @@ class InstantNGP(DataParser): ) # TODO(ethan): add alpha background color - dataset_inputs = DataparserOutputs( + dataparser_outputs = DataparserOutputs( image_filenames=image_filenames, cameras=cameras, scene_box=scene_box, ) - return dataset_inputs + return dataparser_outputs diff --git a/nerfstudio/data/dataparsers/mipnerf_dataparser.py b/nerfstudio/data/dataparsers/mipnerf_dataparser.py index 4d85b4f22..71d69695b 100644 --- a/nerfstudio/data/dataparsers/mipnerf_dataparser.py +++ b/nerfstudio/data/dataparsers/mipnerf_dataparser.py @@ -78,7 +78,7 @@ class Mipnerf360(DataParser): poses_orig[:, :3, :4] = poses[:, :3, :4] return poses_orig - def _generate_dataset_inputs(self, split="train"): + def _generate_dataparser_outputs(self, split="train"): image_dir = "images" if self.config.downscale_factor > 1: image_dir += f"_{self.config.downscale_factor}" @@ -155,10 +155,10 @@ class Mipnerf360(DataParser): cameras.rescale_output_resolution(scaling_factor=1.0 / self.config.downscale_factor) - dataset_inputs = DataparserOutputs( + dataparser_outputs = DataparserOutputs( image_filenames=image_filenames, cameras=cameras, scene_box=scene_box, ) - return dataset_inputs + return dataparser_outputs diff --git a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py index 3f445f36e..b78c5dc9d 100644 --- a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py +++ b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py @@ -62,7 +62,7 @@ class Nerfstudio(DataParser): config: NerfstudioDataParserConfig - def _generate_dataset_inputs(self, split="train"): + def _generate_dataparser_outputs(self, split="train"): # pylint: disable=too-many-statements meta = load_from_json(self.config.data_directory / "transforms.json") @@ -157,9 +157,9 @@ class Nerfstudio(DataParser): cameras.rescale_output_resolution(scaling_factor=1.0 / self.config.downscale_factor) - dataset_inputs = DataparserOutputs( + dataparser_outputs = DataparserOutputs( image_filenames=image_filenames, cameras=cameras, scene_box=scene_box, ) - return dataset_inputs + return dataparser_outputs diff --git a/nerfstudio/data/dataparsers/record3d_dataparser.py b/nerfstudio/data/dataparsers/record3d_dataparser.py index 1ef56ae11..4edc5e4c3 100644 --- a/nerfstudio/data/dataparsers/record3d_dataparser.py +++ b/nerfstudio/data/dataparsers/record3d_dataparser.py @@ -60,7 +60,7 @@ class Record3D(DataParser): config: Record3DDataParserConfig - def _generate_dataset_inputs(self, split: str = "train") -> DataparserOutputs: + def _generate_dataparser_outputs(self, split: str = "train") -> DataparserOutputs: image_dir = self.config.data_directory / "rgb" if not image_dir.exists(): @@ -144,10 +144,10 @@ class Record3D(DataParser): camera_type=CameraType.PERSPECTIVE, ) - dataset_inputs = DataparserOutputs( + dataparser_outputs = DataparserOutputs( image_filenames=image_filenames, cameras=cameras, scene_box=scene_box, ) - return dataset_inputs + return dataparser_outputs diff --git a/nerfstudio/data/utils/dataloaders.py b/nerfstudio/data/utils/dataloaders.py index 96c0c227d..a0d84fe01 100644 --- a/nerfstudio/data/utils/dataloaders.py +++ b/nerfstudio/data/utils/dataloaders.py @@ -116,7 +116,7 @@ class EvalDataloader(DataLoader): **kwargs, ): self.input_dataset = input_dataset - self.cameras = input_dataset.dataset_inputs.cameras.to(device) + self.cameras = input_dataset.dataparser_outputs.cameras.to(device) self.device = device self.kwargs = kwargs super().__init__(dataset=input_dataset) diff --git a/nerfstudio/data/utils/datasets.py b/nerfstudio/data/utils/datasets.py index 6e471272e..689b26f30 100644 --- a/nerfstudio/data/utils/datasets.py +++ b/nerfstudio/data/utils/datasets.py @@ -33,15 +33,15 @@ class InputDataset(Dataset): """Dataset that returns images. Args: - dataset_inputs: description of where and how to read input images. + dataparser_outputs: description of where and how to read input images. """ - def __init__(self, dataset_inputs: DataparserOutputs): + def __init__(self, dataparser_outputs: DataparserOutputs): super().__init__() - self.dataset_inputs = dataset_inputs + self.dataparser_outputs = dataparser_outputs def __len__(self): - return len(self.dataset_inputs.image_filenames) + return len(self.dataparser_outputs.image_filenames) def get_numpy_image(self, image_idx: int) -> npt.NDArray[np.uint8]: """Returns the image of shape (H, W, 3 or 4). @@ -49,7 +49,7 @@ class InputDataset(Dataset): Args: image_idx: The image index in the dataset. """ - image_filename = self.dataset_inputs.image_filenames[image_idx] + image_filename = self.dataparser_outputs.image_filenames[image_idx] pil_image = Image.open(image_filename) image = np.array(pil_image, dtype="uint8") # shape is (h, w, 3 or 4) assert len(image.shape) == 3 @@ -64,9 +64,9 @@ class InputDataset(Dataset): image_idx: The image index in the dataset. """ image = torch.from_numpy(self.get_numpy_image(image_idx).astype("float32") / 255.0) - if self.dataset_inputs.alpha_color is not None and image.shape[-1] == 4: + if self.dataparser_outputs.alpha_color is not None and image.shape[-1] == 4: assert image.shape[-1] == 4 - image = image[:, :, :3] * image[:, :, -1:] + self.dataset_inputs.alpha_color * (1.0 - image[:, :, -1:]) + image = image[:, :, :3] * image[:, :, -1:] + self.dataparser_outputs.alpha_color * (1.0 - image[:, :, -1:]) else: image = image[:, :, :3] return image @@ -80,7 +80,7 @@ class InputDataset(Dataset): image = self.get_image(image_idx) data = {"image_idx": image_idx} data["image"] = image - for _, data_func_dict in self.dataset_inputs.additional_inputs.items(): + for _, data_func_dict in self.dataparser_outputs.additional_inputs.items(): assert "func" in data_func_dict, "Missing function to process data: specify `func` in `additional_inputs`" func = data_func_dict["func"] assert "kwargs" in data_func_dict, "No data to process: specify `kwargs` in `additional_inputs`" diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py index 56a62ba11..b771922d7 100644 --- a/nerfstudio/pipelines/base_pipeline.py +++ b/nerfstudio/pipelines/base_pipeline.py @@ -218,7 +218,7 @@ class VanillaPipeline(Pipeline): # TODO(ethan): get rid of scene_bounds from the model assert self.datamanager.train_dataset is not None, "Missing input dataset" self._model = config.model.setup( - scene_box=self.datamanager.train_dataset.dataset_inputs.scene_box, + scene_box=self.datamanager.train_dataset.dataparser_outputs.scene_box, num_train_data=len(self.datamanager.train_dataset), ) self.model.to(device) diff --git a/nerfstudio/viewer/server/viewer_utils.py b/nerfstudio/viewer/server/viewer_utils.py index 6a6446f04..21066e6ff 100644 --- a/nerfstudio/viewer/server/viewer_utils.py +++ b/nerfstudio/viewer/server/viewer_utils.py @@ -272,11 +272,11 @@ class ViewerState: for idx in image_indices: image = dataset[idx]["image"] bgr = image[..., [2, 1, 0]] - camera_json = dataset.dataset_inputs.cameras.to_json(camera_idx=idx, image=bgr, max_size=100) + camera_json = dataset.dataparser_outputs.cameras.to_json(camera_idx=idx, image=bgr, max_size=100) self.vis[f"sceneState/cameras/{idx:06d}"].write(camera_json) # draw the scene box (i.e., the bounding box) - json_ = dataset.dataset_inputs.scene_box.to_json() + json_ = dataset.dataparser_outputs.scene_box.to_json() self.vis["sceneState/sceneBox"].write(json_) # set the initial state whether to train or not