diff --git a/.gitignore b/.gitignore index ab8a4e108..cfa4358d2 100644 --- a/.gitignore +++ b/.gitignore @@ -171,6 +171,7 @@ events.out.* # Data data !*/data +!docs/reference/api/data/ # Misc old/ diff --git a/docs/conf.py b/docs/conf.py index 2071057b9..824a61d78 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -42,6 +42,7 @@ extensions = [ "sphinx_design", "sphinx.ext.mathjax", "sphinxext.opengraph", + "sphinx.ext.viewcode", ] # List of patterns, relative to source directory, that match files and diff --git a/docs/developer_guides/pipelines/dataparsers.md b/docs/developer_guides/pipelines/dataparsers.md index 3a38664f9..19b964e5a 100644 --- a/docs/developer_guides/pipelines/dataparsers.md +++ b/docs/developer_guides/pipelines/dataparsers.md @@ -30,8 +30,16 @@ class DataparserOutputs: """Color of dataset background.""" scene_box: SceneBox = SceneBox() """Scene box of dataset. Used to bound the scene or provide the scene scale depending on model.""" - semantics: Optional[Semantics] = None - """Semantics information.""" + mask_filenames: Optional[List[Path]] = None + """Filenames for any masks that are required""" + metadata: Dict[str, Any] = to_immutable_dict({}) + """Dictionary of any metadata that be required for the given experiment. + Will be processed by the InputDataset to create any additional tensors that may be required. + """ + dataparser_transform: TensorType[3, 4] = torch.eye(4)[:3, :] + """Transform applied by the dataparser.""" + dataparser_scale: float = 1.0 + """Scale applied by the dataparser.""" @dataclass class DataParser: @@ -115,48 +123,12 @@ You can also pull out information from the DataParserOutputs for other DataManga ray_generator = RayGenerator(dataparser_outputs.cameras) ``` -## Our Implementations +## Included DataParsers -Below we enumerate the various dataparsers that we have implemented in our codebase. Feel free to use ours or add your own. Also any contributions are welcome and appreciated! +```{toctree} +--- +maxdepth: 2 +--- -###### Nerfstudio - -This is our custom dataparser. We have a script to convert images or videos with COLMAP to this format. - -```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/nerfstudio_dataparser.py -:color: primary -:outline: -See the code! -``` - -###### Blender - -We support the synthetic Blender dataset from the original NeRF paper. - -```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/blender_dataparser.py -:color: primary -:outline: -See the code! -``` - -###### Instant NGP - -This supports the Instant NGP dataset. - -```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/instant_ngp_dataparser.py -:color: primary -:outline: -See the code! -``` - -###### Record3D - -This dataparser can use recorded data from a >= iPhone 12 Pro using the [Record3D app](https://record3d.app/) . Record a video and export with the `EXR + JPG sequence` format. Unzip export and `rgb` folder before training. - -For more information on capturing with Record3D, see the [Custom Dataset Docs](/quickstart/custom_dataset.md). - -```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/record3d_dataparser.py -:color: primary -:outline: -See the code! +../../reference/api/data/dataparsers ``` diff --git a/docs/reference/api/data.rst b/docs/reference/api/data.rst deleted file mode 100644 index de7b57abe..000000000 --- a/docs/reference/api/data.rst +++ /dev/null @@ -1,39 +0,0 @@ -.. _dataset: - -Data -============ - -Data Parsers ----------------- - -.. automodule:: nerfstudio.data.dataparsers - :members: - :show-inheritance: - -Datasets ----------------- - -.. automodule:: nerfstudio.data.datasets - :members: - :show-inheritance: - -Dataloaders ----------------- - -.. automodule:: nerfstudio.data.utils.dataloaders - :members: - :show-inheritance: - -Pixel Samplers ----------------- - -.. automodule:: nerfstudio.data.pixel_samplers - :members: - :show-inheritance: - -Scene Box ----------------- - -.. automodule:: nerfstudio.data.scene_box - :members: - :show-inheritance: \ No newline at end of file diff --git a/docs/reference/api/data/datamanagers.rst b/docs/reference/api/data/datamanagers.rst new file mode 100644 index 000000000..7779c9e10 --- /dev/null +++ b/docs/reference/api/data/datamanagers.rst @@ -0,0 +1,31 @@ +.. _datamanagers: + +Datamanagers +============ + +Base +---------------- + +.. automodule:: nerfstudio.data.datamanagers.base_datamanager + :members: + +Depth Datamanager +----------------- + +.. automodule:: nerfstudio.data.datamanagers.depth_datamanager + :members: + :show-inheritance: + +SDF Datamanager +---------------- + +.. automodule:: nerfstudio.data.datamanagers.sdf_datamanager + :members: + :show-inheritance: + +Semantic Datamanager +--------------------- + +.. automodule:: nerfstudio.data.datamanagers.semantic_datamanager + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/reference/api/data/dataparsers.rst b/docs/reference/api/data/dataparsers.rst new file mode 100644 index 000000000..e35f83ea0 --- /dev/null +++ b/docs/reference/api/data/dataparsers.rst @@ -0,0 +1,117 @@ +.. _dataparser: + +Data Parsers +============ + +* `Base Data Parser`_ +* `ARKitScenes`_ +* `Blender`_ +* `D-NeRF`_ +* `dycheck`_ +* `Instant-NGP`_ +* `Minimal`_ +* `NeRF-OSR`_ +* `Nerfstudio`_ +* `nuScenes`_ +* `Phototourism`_ +* `ScanNet`_ +* `SDFStudio`_ +* `sitcoms3D`_ + + +Base Data Parser +---------------- + +.. automodule:: nerfstudio.data.dataparsers.base_dataparser + :members: + +ARKitScenes +---------------- + +.. automodule:: nerfstudio.data.dataparsers.arkitscenes_dataparser + :members: + :show-inheritance: + +Blender +---------------- + +.. automodule:: nerfstudio.data.dataparsers.blender_dataparser + :members: + :show-inheritance: + +D-NeRF +---------------- + +.. automodule:: nerfstudio.data.dataparsers.dnerf_dataparser + :members: + :show-inheritance: + +dycheck +---------------- + +.. automodule:: nerfstudio.data.dataparsers.dycheck_dataparser + :members: + :show-inheritance: + +Instant-NGP +---------------- + +.. automodule:: nerfstudio.data.dataparsers.instant_ngp_dataparser + :members: + :show-inheritance: + +Minimal +---------------- + +.. automodule:: nerfstudio.data.dataparsers.minimal_dataparser + :members: + :show-inheritance: + +NeRF-OSR +---------------- + +.. automodule:: nerfstudio.data.dataparsers.nerfosr_dataparser + :members: + :show-inheritance: + +Nerfstudio +---------------- + +.. automodule:: nerfstudio.data.dataparsers.nerfstudio_dataparser + :members: + :show-inheritance: + +nuScenes +---------------- + +.. automodule:: nerfstudio.data.dataparsers.nuscenes_dataparser + :members: + :show-inheritance: + +Phototourism +---------------- + +.. automodule:: nerfstudio.data.dataparsers.phototourism_dataparser + :members: + :show-inheritance: + +ScanNet +---------------- + +.. automodule:: nerfstudio.data.dataparsers.scannet_dataparser + :members: + :show-inheritance: + +SDFStudio +---------------- + +.. automodule:: nerfstudio.data.dataparsers.sdfstudio_dataparser + :members: + :show-inheritance: + +sitcoms3D +---------------- + +.. automodule:: nerfstudio.data.dataparsers.sitcoms3d_dataparser + :members: + :show-inheritance: diff --git a/docs/reference/api/data/datasets.rst b/docs/reference/api/data/datasets.rst new file mode 100644 index 000000000..ee4fd1ab5 --- /dev/null +++ b/docs/reference/api/data/datasets.rst @@ -0,0 +1,24 @@ +.. _datasets: + +Datasets +============ + +Base +---------------- + +.. automodule:: nerfstudio.data.datasets.base_dataset + :members: + +SDF Dataset +---------------- + +.. automodule:: nerfstudio.data.datasets.sdf_dataset + :members: + :show-inheritance: + +Semantic Dataset +---------------- + +.. automodule:: nerfstudio.data.datasets.semantic_dataset + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/reference/api/data/index.rst b/docs/reference/api/data/index.rst new file mode 100644 index 000000000..6e10f9384 --- /dev/null +++ b/docs/reference/api/data/index.rst @@ -0,0 +1,26 @@ +.. _dataset: + +Data +============ + +.. toctree:: + :titlesonly: + + dataparsers + datamanagers + datasets + utils + +Pixel Samplers +---------------- + +.. automodule:: nerfstudio.data.pixel_samplers + :members: + :show-inheritance: + +Scene Box +---------------- + +.. automodule:: nerfstudio.data.scene_box + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/reference/api/data/utils.rst b/docs/reference/api/data/utils.rst new file mode 100644 index 000000000..5f225e6da --- /dev/null +++ b/docs/reference/api/data/utils.rst @@ -0,0 +1,28 @@ +.. _datautils: + +Utils +============ + +Base +---------------- + +.. automodule:: nerfstudio.data.utils.colmap_parsing_utils + :members: + +Data +---------------- + +.. automodule:: nerfstudio.data.utils.data_utils + :members: + +Dataloader +---------------- + +.. automodule:: nerfstudio.data.utils.dataloaders + :members: + +Nerfstudio Collate +------------------- + +.. automodule:: nerfstudio.data.utils.nerfstudio_collate + :members: \ No newline at end of file diff --git a/docs/reference/api/index.rst b/docs/reference/api/index.rst index c1c9ce0b2..01c7d7fe7 100644 --- a/docs/reference/api/index.rst +++ b/docs/reference/api/index.rst @@ -8,7 +8,7 @@ TODO: Explanation of each component .. toctree:: config - data + data/index fields field_components/index models diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index 8bcd21bbe..7e2ebedeb 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -149,10 +149,11 @@ class DataManager(nn.Module): Usage: To get data, use the next_train and next_eval functions. This data manager's next_train and next_eval methods will return 2 things: - 1. A Raybundle: This will contain the rays we are sampling, with latents and - conditionals attached (everything needed at inference) - 2. A "batch" of auxiliary information: This will contain the mask, the ground truth - pixels, etc needed to actually train, score, etc the model + + 1. A Raybundle: This will contain the rays we are sampling, with latents and + conditionals attached (everything needed at inference) + 2. A "batch" of auxiliary information: This will contain the mask, the ground truth + pixels, etc needed to actually train, score, etc the model Rationale: Because of this abstraction we've added, we can support more NeRF paradigms beyond the diff --git a/nerfstudio/data/dataparsers/nerfosr_dataparser.py b/nerfstudio/data/dataparsers/nerfosr_dataparser.py index 820a28956..f7b84b261 100644 --- a/nerfstudio/data/dataparsers/nerfosr_dataparser.py +++ b/nerfstudio/data/dataparsers/nerfosr_dataparser.py @@ -42,7 +42,7 @@ from nerfstudio.data.scene_box import SceneBox CONSOLE = Console(width=120) -def find_files(directory: str, exts: List[str]): +def _find_files(directory: str, exts: List[str]): """Find all files in a directory that have a certain file extension. Args: @@ -94,8 +94,8 @@ def get_camera_params( split_dir = f"{scene_dir}/{split}" # camera parameters files - intrinsics_files = find_files(f"{split_dir}/intrinsics", exts=["*.txt"]) - pose_files = find_files(f"{split_dir}/pose", exts=["*.txt"]) + intrinsics_files = _find_files(f"{split_dir}/intrinsics", exts=["*.txt"]) + pose_files = _find_files(f"{split_dir}/pose", exts=["*.txt"]) num_cams = len(pose_files) @@ -219,12 +219,12 @@ class NeRFOSR(DataParser): ) # --- images --- - image_filenames = find_files(f"{split_dir}/rgb", exts=["*.png", "*.jpg", "*.JPG", "*.PNG"]) + image_filenames = _find_files(f"{split_dir}/rgb", exts=["*.png", "*.jpg", "*.JPG", "*.PNG"]) # --- masks --- mask_filenames = [] if self.config.use_masks: - mask_filenames = find_files(f"{split_dir}/mask", exts=["*.png", "*.jpg", "*.JPG", "*.PNG"]) + mask_filenames = _find_files(f"{split_dir}/mask", exts=["*.png", "*.jpg", "*.JPG", "*.PNG"]) dataparser_outputs = DataparserOutputs( image_filenames=image_filenames, diff --git a/nerfstudio/data/dataparsers/scannet_dataparser.py b/nerfstudio/data/dataparsers/scannet_dataparser.py index d5a787799..5d2702f18 100644 --- a/nerfstudio/data/dataparsers/scannet_dataparser.py +++ b/nerfstudio/data/dataparsers/scannet_dataparser.py @@ -39,10 +39,14 @@ class ScanNetDataParserConfig(DataParserConfig): ScanNet dataset (https://www.scan-net.org/) is a large-scale 3D dataset of indoor scenes. This dataparser assumes that the dense stream was extracted from .sens files. Expected structure of scene directory: - - color/ - - depth/ - - intrinsic/ - - pose/ + + .. code-block:: text + + root/ + ├── color/ + ├── depth/ + ├── intrinsic/ + ├── pose/ """ _target: Type = field(default_factory=lambda: ScanNet)