From 4e6c52c8fdbcd520027585a58d8103f1edf3f619 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Tue, 4 Apr 2023 11:08:09 -0700 Subject: [PATCH 1/5] Update sphinx docs (#1695) --- docs/quickstart/custom_dataset.md | 21 +++++++++++++++++++++ docs/reference/cli/index.md | 21 +++++++++++---------- docs/reference/cli/ns_install_cli.md | 9 +++++++++ scripts/completions/install.py | 3 +++ 4 files changed, 44 insertions(+), 10 deletions(-) create mode 100644 docs/reference/cli/ns_install_cli.md diff --git a/docs/quickstart/custom_dataset.md b/docs/quickstart/custom_dataset.md index 09e4d1930..65e6c22d4 100644 --- a/docs/quickstart/custom_dataset.md +++ b/docs/quickstart/custom_dataset.md @@ -20,6 +20,7 @@ We Currently support the following custom data types: | 📱 [KIRI Engine](kiri) | IOS or Android | [KIRI Engine App](https://www.kiriengine.com/) | 🐇 | | 📱 [Record3D](record3d) | IOS with LiDAR | [Record3D app](https://record3d.app/) | 🐇 | | 🖥 [Metashape](metashape) | Any | [Metashape](https://www.agisoft.com/) | 🐇 | +| 🖥 [RealityCapture](realitycapture) | Any | [RealityCapture](https://www.capturingreality.com/realitycapture) | 🐇 | (images_and_video)= @@ -286,6 +287,26 @@ ns-process-data metashape --data {data directory} --xml {xml file} --output-dir ns-train nerfacto --data {output directory} ``` +(realitycapture)= + +## RealityCapture + +1. Align your images using RealityCapture. `ALIGNMENT -> Align Images` + +2. Export the camera alignment as a `csv` file. Choose `Internal/External camera parameters` + +3. Convert the data to the nerfstudio format. + +```bash +ns-process-data realitycapture --data {data directory} --csv {csv file} --output-dir {output directory} +``` + +4. Train with nerfstudio! + +```bash +ns-train nerfacto --data {output directory} +``` + (360_data)= ## 360 Data (Equirectangular) diff --git a/docs/reference/cli/index.md b/docs/reference/cli/index.md index 9e1b8fae4..7bc95834f 100644 --- a/docs/reference/cli/index.md +++ b/docs/reference/cli/index.md @@ -6,21 +6,22 @@ We provide a command line interface for training your own NeRFs (no coding neces Here are the popular commands that we offer. If you've cloned the repo, you can also look at the [pyproject.toml file](https://github.com/nerfstudio-project/nerfstudio/blob/main/pyproject.toml) at the `[project.scripts]` section for details. -| Command | Description | Filename | -| ---------------- | -------------------------------------- | ---------------------------------- | -| ns-install-cli | Install tab completion for all scripts | scripts/completions/install.py | -| ns-process-data | Generate a dataset from your own data | scripts/process_data.py | -| ns-download-data | Download existing captures | scripts/downloads/download_data.py | -| ns-train | Generate a NeRF | scripts/train.py | -| ns-viewer | View a trained NeRF | scripts/viewer/run_viewer.py | -| ns-eval | Run evaluation metrics for your Model | scripts/eval.py | -| ns-render | Render out a video of your NeRF | scripts/render.py | -| ns-export | Export a NeRF into other formats | scripts/exporter.py | +| Command | Description | Filename | +| ------------------------------------ | -------------------------------------- | ---------------------------------- | +| [ns-install-cli](ns_install_cli) | Install tab completion for all scripts | scripts/completions/install.py | +| [ns-process-data](ns_process_data) | Generate a dataset from your own data | scripts/process_data.py | +| [ns-download-data](ns_download_data) | Download existing captures | scripts/downloads/download_data.py | +| [ns-train](ns_train) | Generate a NeRF | scripts/train.py | +| [ns-viewer](ns_viewer) | View a trained NeRF | scripts/viewer/run_viewer.py | +| [ns-eval](ns_eval) | Run evaluation metrics for your Model | scripts/eval.py | +| [ns-render](ns_render) | Render out a video of your NeRF | scripts/render.py | +| [ns-export](ns_export) | Export a NeRF into other formats | scripts/exporter.py | ```{toctree} :maxdepth: 1 :hidden: +ns_install_cli ns_process_data ns_download_data ns_train diff --git a/docs/reference/cli/ns_install_cli.md b/docs/reference/cli/ns_install_cli.md new file mode 100644 index 000000000..e9c1a9541 --- /dev/null +++ b/docs/reference/cli/ns_install_cli.md @@ -0,0 +1,9 @@ +# ns-install-cli + +```{eval-rst} +.. argparse:: + :module: scripts.completions.install + :func: get_parser_fn + :prog: ns-install-cli + :nodefault: +``` diff --git a/scripts/completions/install.py b/scripts/completions/install.py index 98e67fea2..067bea2bd 100644 --- a/scripts/completions/install.py +++ b/scripts/completions/install.py @@ -399,3 +399,6 @@ def entrypoint(): if __name__ == "__main__": entrypoint() + +# For sphinx docs +get_parser_fn = lambda: tyro.extras.get_parser(main) # noqa From 7126334976119ff5ebd3356ada0449de11458f10 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Tue, 4 Apr 2023 12:49:15 -0700 Subject: [PATCH 2/5] Docs for adding new methods (#1697) --- README.md | 21 ++++---- docs/developer_guides/config.md | 30 ----------- docs/developer_guides/new_methods.md | 75 ++++++++++++++++++++++++++++ docs/index.md | 1 + 4 files changed, 87 insertions(+), 40 deletions(-) create mode 100644 docs/developer_guides/new_methods.md diff --git a/README.md b/README.md index ddd0873e0..57d77724b 100644 --- a/README.md +++ b/README.md @@ -196,16 +196,17 @@ ns-export pointcloud --help Using an existing dataset is great, but likely you want to use your own data! We support various methods for using your own data. Before it can be used in nerfstudio, the camera location and orientations must be determined and then converted into our format using `ns-process-data`. We rely on external tools for this, instructions and information can be found in the documentation. -| Data | Capture Device | Requirements | `ns-process-data` Speed | -| ------------------------------------------------------------------------------------------------- | -------------- | ----------------------------------------------- | ----------------------- | -| 📷 [Images](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#images-and-video) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐢 | -| 📹 [Video](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#images-and-video) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐢 | -| 🌎 [360 Data](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#360_data) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐢 | -| 📱 [Polycam](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#polycam-capture) | IOS with LiDAR | [Polycam App](https://poly.cam/) | 🐇 | -| 📱 [KIRI Engine](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#kiri-capture) | IOS or Android | [KIRI Engine App](https://www.kiriengine.com/) | 🐇 | -| 📱 [Record3D](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#record3d-capture) | IOS with LiDAR | [Record3D app](https://record3d.app/) | 🐇 | -| 🖥 [Metashape](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#metashape) | Any | [Metashape](https://www.agisoft.com/) | 🐇 | -| 🛠 [Custom](https://docs.nerf.studio/en/latest/quickstart/data_conventions.html) | Any | Camera Poses | 🐇 | +| Data | Capture Device | Requirements | `ns-process-data` Speed | +| ---------------------------------------------------------------------------------------------------- | -------------- | ----------------------------------------------------------------- | ----------------------- | +| 📷 [Images](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#images-and-video) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐢 | +| 📹 [Video](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#images-and-video) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐢 | +| 🌎 [360 Data](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#360_data) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐢 | +| 📱 [Polycam](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#polycam-capture) | IOS with LiDAR | [Polycam App](https://poly.cam/) | 🐇 | +| 📱 [KIRI Engine](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#kiri-capture) | IOS or Android | [KIRI Engine App](https://www.kiriengine.com/) | 🐇 | +| 📱 [Record3D](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#record3d-capture) | IOS with LiDAR | [Record3D app](https://record3d.app/) | 🐇 | +| 🖥 [Metashape](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#metashape) | Any | [Metashape](https://www.agisoft.com/) | 🐇 | +| 🖥 [RealityCapture](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#realitycapture) | Any | [RealityCapture](https://www.capturingreality.com/realitycapture) | 🐇 | +| 🛠 [Custom](https://docs.nerf.studio/en/latest/quickstart/data_conventions.html) | Any | Camera Poses | 🐇 | ## 5. Advanced Options diff --git a/docs/developer_guides/config.md b/docs/developer_guides/config.md index 29d093f9d..bf216278f 100644 --- a/docs/developer_guides/config.md +++ b/docs/developer_guides/config.md @@ -118,33 +118,3 @@ Often times, you just want to play with the parameters of an existing model with # NOTE: the dataparser and associated configurations go at the end of the command ns-train {METHOD_NAME} --vis viewer {DATA_PARSER} --scale-factor 0.5 ``` - -### Extending Nerfstudio with custom methods -In order to extend the Nerfstudio and register your own methods, you can package your code as a python package -and register it with Nerfstudio as a `nerfstudio.method_configs` entrypoint in the `pyproject.toml` file. -The Nerfstudio will automatically look for all registered methods and will register them to be used -by method such as `ns-train`. - -Here is an example: -```python -"""my_package/my_config.py""" - -from nerfstudio.engine.trainer import TrainerConfig -from nerfstudio.plugins.types import MethodSpecification - -MyMethod = MethodSpecification( - config=TrainerConfig( - method_name="my-method", - ... - ), - description="Custom description" -) - -"""pyproject.toml""" -[project] -name = "my_package" -... - -[project.entry-points.'nerfstudio.method_configs'] -my-method = 'my_package.my_config:MyMethod' -``` \ No newline at end of file diff --git a/docs/developer_guides/new_methods.md b/docs/developer_guides/new_methods.md new file mode 100644 index 000000000..c0484ca9e --- /dev/null +++ b/docs/developer_guides/new_methods.md @@ -0,0 +1,75 @@ +# Adding a New Method + +Nerfstudio aims to offer researchers a codebase that they can utilize to extend and develop novel methods. Our vision is for users to establish a distinct repository that imports nerfstudio and overrides pipeline components to cater to specific functionality requirements of the new approach. If any of the new features require modifications to the core nerfstudio repository and can be generally useful, we encourage you to submit a PR to enable others to benefit from it. + +Examples are often the best way to learn, take a look at the [LERF](https://github.com/kerrj/lerf) repository for good example of how to use nerfstudio in your projects. + +## File Structure + +We recommend the following file structure: + +``` +├── my_method +│ ├── my_config.py +│ ├── custom_pipeline.py [optional] +│ ├── custom_model.py [optional] +│ ├── custom_field.py [optional] +│ ├── custom_datamanger.py [optional] +│ ├── ... +├── pyproject.toml +``` + +## Registering custom model with nerfstudio + +In order to extend the Nerfstudio and register your own methods, you can package your code as a python package +and register it with Nerfstudio as a `nerfstudio.method_configs` entrypoint in the `pyproject.toml` file. +Nerfstudio will automatically look for all registered methods and will register them to be used +by methods such as `ns-train`. + +First create a config file: + +```python +"""my_method/my_config.py""" + +from nerfstudio.engine.trainer import TrainerConfig +from nerfstudio.plugins.types import MethodSpecification + +MyMethod = MethodSpecification( + config=TrainerConfig( + method_name="my-method", + pipeline=... + ... + ), + description="Custom description" +) +``` + +Then create a `pyproject.toml` file. This is where the entrypoint to your method is set and also where you can specify additional dependencies required by your codebase. + +```python +"""pyproject.toml""" + +[project] +name = "my_method" + +dependencies = [ + "nerfstudio" # you may want to consider pinning the version, ie "nerfstudio==0.1.19" +] + +[project.entry-points.'nerfstudio.method_configs'] +my-method = 'my_method.my_config:MyMethod' +``` + +finally run the following to register the method, + +``` +pip install -e . +``` + +## Running custom method + +After registering your method you should be able to run the method with, + +``` +ns-train my-method --data DATA_DIR +``` diff --git a/docs/index.md b/docs/index.md index b373097e5..d28393883 100644 --- a/docs/index.md +++ b/docs/index.md @@ -75,6 +75,7 @@ nerfology/model_components/index :hidden: :caption: Developer Guides +developer_guides/new_methods developer_guides/pipelines/index developer_guides/viewer/viewer_overview developer_guides/config From 2e777e457746b8c1afc800ea6eea05e84066a54f Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Tue, 4 Apr 2023 18:30:52 -0700 Subject: [PATCH 3/5] Update dataparser docs (#1698) * Update dataparser docs * Add missing files --- .gitignore | 1 + docs/conf.py | 1 + .../developer_guides/pipelines/dataparsers.md | 60 +++------ docs/reference/api/data.rst | 39 ------ docs/reference/api/data/datamanagers.rst | 31 +++++ docs/reference/api/data/dataparsers.rst | 117 ++++++++++++++++++ docs/reference/api/data/datasets.rst | 24 ++++ docs/reference/api/data/index.rst | 26 ++++ docs/reference/api/data/utils.rst | 28 +++++ docs/reference/api/index.rst | 2 +- .../data/datamanagers/base_datamanager.py | 9 +- .../data/dataparsers/nerfosr_dataparser.py | 10 +- .../data/dataparsers/scannet_dataparser.py | 12 +- 13 files changed, 263 insertions(+), 97 deletions(-) delete mode 100644 docs/reference/api/data.rst create mode 100644 docs/reference/api/data/datamanagers.rst create mode 100644 docs/reference/api/data/dataparsers.rst create mode 100644 docs/reference/api/data/datasets.rst create mode 100644 docs/reference/api/data/index.rst create mode 100644 docs/reference/api/data/utils.rst diff --git a/.gitignore b/.gitignore index ab8a4e108..cfa4358d2 100644 --- a/.gitignore +++ b/.gitignore @@ -171,6 +171,7 @@ events.out.* # Data data !*/data +!docs/reference/api/data/ # Misc old/ diff --git a/docs/conf.py b/docs/conf.py index 2071057b9..824a61d78 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -42,6 +42,7 @@ extensions = [ "sphinx_design", "sphinx.ext.mathjax", "sphinxext.opengraph", + "sphinx.ext.viewcode", ] # List of patterns, relative to source directory, that match files and diff --git a/docs/developer_guides/pipelines/dataparsers.md b/docs/developer_guides/pipelines/dataparsers.md index 3a38664f9..19b964e5a 100644 --- a/docs/developer_guides/pipelines/dataparsers.md +++ b/docs/developer_guides/pipelines/dataparsers.md @@ -30,8 +30,16 @@ class DataparserOutputs: """Color of dataset background.""" scene_box: SceneBox = SceneBox() """Scene box of dataset. Used to bound the scene or provide the scene scale depending on model.""" - semantics: Optional[Semantics] = None - """Semantics information.""" + mask_filenames: Optional[List[Path]] = None + """Filenames for any masks that are required""" + metadata: Dict[str, Any] = to_immutable_dict({}) + """Dictionary of any metadata that be required for the given experiment. + Will be processed by the InputDataset to create any additional tensors that may be required. + """ + dataparser_transform: TensorType[3, 4] = torch.eye(4)[:3, :] + """Transform applied by the dataparser.""" + dataparser_scale: float = 1.0 + """Scale applied by the dataparser.""" @dataclass class DataParser: @@ -115,48 +123,12 @@ You can also pull out information from the DataParserOutputs for other DataManga ray_generator = RayGenerator(dataparser_outputs.cameras) ``` -## Our Implementations +## Included DataParsers -Below we enumerate the various dataparsers that we have implemented in our codebase. Feel free to use ours or add your own. Also any contributions are welcome and appreciated! +```{toctree} +--- +maxdepth: 2 +--- -###### Nerfstudio - -This is our custom dataparser. We have a script to convert images or videos with COLMAP to this format. - -```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/nerfstudio_dataparser.py -:color: primary -:outline: -See the code! -``` - -###### Blender - -We support the synthetic Blender dataset from the original NeRF paper. - -```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/blender_dataparser.py -:color: primary -:outline: -See the code! -``` - -###### Instant NGP - -This supports the Instant NGP dataset. - -```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/instant_ngp_dataparser.py -:color: primary -:outline: -See the code! -``` - -###### Record3D - -This dataparser can use recorded data from a >= iPhone 12 Pro using the [Record3D app](https://record3d.app/) . Record a video and export with the `EXR + JPG sequence` format. Unzip export and `rgb` folder before training. - -For more information on capturing with Record3D, see the [Custom Dataset Docs](/quickstart/custom_dataset.md). - -```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/record3d_dataparser.py -:color: primary -:outline: -See the code! +../../reference/api/data/dataparsers ``` diff --git a/docs/reference/api/data.rst b/docs/reference/api/data.rst deleted file mode 100644 index de7b57abe..000000000 --- a/docs/reference/api/data.rst +++ /dev/null @@ -1,39 +0,0 @@ -.. _dataset: - -Data -============ - -Data Parsers ----------------- - -.. automodule:: nerfstudio.data.dataparsers - :members: - :show-inheritance: - -Datasets ----------------- - -.. automodule:: nerfstudio.data.datasets - :members: - :show-inheritance: - -Dataloaders ----------------- - -.. automodule:: nerfstudio.data.utils.dataloaders - :members: - :show-inheritance: - -Pixel Samplers ----------------- - -.. automodule:: nerfstudio.data.pixel_samplers - :members: - :show-inheritance: - -Scene Box ----------------- - -.. automodule:: nerfstudio.data.scene_box - :members: - :show-inheritance: \ No newline at end of file diff --git a/docs/reference/api/data/datamanagers.rst b/docs/reference/api/data/datamanagers.rst new file mode 100644 index 000000000..7779c9e10 --- /dev/null +++ b/docs/reference/api/data/datamanagers.rst @@ -0,0 +1,31 @@ +.. _datamanagers: + +Datamanagers +============ + +Base +---------------- + +.. automodule:: nerfstudio.data.datamanagers.base_datamanager + :members: + +Depth Datamanager +----------------- + +.. automodule:: nerfstudio.data.datamanagers.depth_datamanager + :members: + :show-inheritance: + +SDF Datamanager +---------------- + +.. automodule:: nerfstudio.data.datamanagers.sdf_datamanager + :members: + :show-inheritance: + +Semantic Datamanager +--------------------- + +.. automodule:: nerfstudio.data.datamanagers.semantic_datamanager + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/reference/api/data/dataparsers.rst b/docs/reference/api/data/dataparsers.rst new file mode 100644 index 000000000..e35f83ea0 --- /dev/null +++ b/docs/reference/api/data/dataparsers.rst @@ -0,0 +1,117 @@ +.. _dataparser: + +Data Parsers +============ + +* `Base Data Parser`_ +* `ARKitScenes`_ +* `Blender`_ +* `D-NeRF`_ +* `dycheck`_ +* `Instant-NGP`_ +* `Minimal`_ +* `NeRF-OSR`_ +* `Nerfstudio`_ +* `nuScenes`_ +* `Phototourism`_ +* `ScanNet`_ +* `SDFStudio`_ +* `sitcoms3D`_ + + +Base Data Parser +---------------- + +.. automodule:: nerfstudio.data.dataparsers.base_dataparser + :members: + +ARKitScenes +---------------- + +.. automodule:: nerfstudio.data.dataparsers.arkitscenes_dataparser + :members: + :show-inheritance: + +Blender +---------------- + +.. automodule:: nerfstudio.data.dataparsers.blender_dataparser + :members: + :show-inheritance: + +D-NeRF +---------------- + +.. automodule:: nerfstudio.data.dataparsers.dnerf_dataparser + :members: + :show-inheritance: + +dycheck +---------------- + +.. automodule:: nerfstudio.data.dataparsers.dycheck_dataparser + :members: + :show-inheritance: + +Instant-NGP +---------------- + +.. automodule:: nerfstudio.data.dataparsers.instant_ngp_dataparser + :members: + :show-inheritance: + +Minimal +---------------- + +.. automodule:: nerfstudio.data.dataparsers.minimal_dataparser + :members: + :show-inheritance: + +NeRF-OSR +---------------- + +.. automodule:: nerfstudio.data.dataparsers.nerfosr_dataparser + :members: + :show-inheritance: + +Nerfstudio +---------------- + +.. automodule:: nerfstudio.data.dataparsers.nerfstudio_dataparser + :members: + :show-inheritance: + +nuScenes +---------------- + +.. automodule:: nerfstudio.data.dataparsers.nuscenes_dataparser + :members: + :show-inheritance: + +Phototourism +---------------- + +.. automodule:: nerfstudio.data.dataparsers.phototourism_dataparser + :members: + :show-inheritance: + +ScanNet +---------------- + +.. automodule:: nerfstudio.data.dataparsers.scannet_dataparser + :members: + :show-inheritance: + +SDFStudio +---------------- + +.. automodule:: nerfstudio.data.dataparsers.sdfstudio_dataparser + :members: + :show-inheritance: + +sitcoms3D +---------------- + +.. automodule:: nerfstudio.data.dataparsers.sitcoms3d_dataparser + :members: + :show-inheritance: diff --git a/docs/reference/api/data/datasets.rst b/docs/reference/api/data/datasets.rst new file mode 100644 index 000000000..ee4fd1ab5 --- /dev/null +++ b/docs/reference/api/data/datasets.rst @@ -0,0 +1,24 @@ +.. _datasets: + +Datasets +============ + +Base +---------------- + +.. automodule:: nerfstudio.data.datasets.base_dataset + :members: + +SDF Dataset +---------------- + +.. automodule:: nerfstudio.data.datasets.sdf_dataset + :members: + :show-inheritance: + +Semantic Dataset +---------------- + +.. automodule:: nerfstudio.data.datasets.semantic_dataset + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/reference/api/data/index.rst b/docs/reference/api/data/index.rst new file mode 100644 index 000000000..6e10f9384 --- /dev/null +++ b/docs/reference/api/data/index.rst @@ -0,0 +1,26 @@ +.. _dataset: + +Data +============ + +.. toctree:: + :titlesonly: + + dataparsers + datamanagers + datasets + utils + +Pixel Samplers +---------------- + +.. automodule:: nerfstudio.data.pixel_samplers + :members: + :show-inheritance: + +Scene Box +---------------- + +.. automodule:: nerfstudio.data.scene_box + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/reference/api/data/utils.rst b/docs/reference/api/data/utils.rst new file mode 100644 index 000000000..5f225e6da --- /dev/null +++ b/docs/reference/api/data/utils.rst @@ -0,0 +1,28 @@ +.. _datautils: + +Utils +============ + +Base +---------------- + +.. automodule:: nerfstudio.data.utils.colmap_parsing_utils + :members: + +Data +---------------- + +.. automodule:: nerfstudio.data.utils.data_utils + :members: + +Dataloader +---------------- + +.. automodule:: nerfstudio.data.utils.dataloaders + :members: + +Nerfstudio Collate +------------------- + +.. automodule:: nerfstudio.data.utils.nerfstudio_collate + :members: \ No newline at end of file diff --git a/docs/reference/api/index.rst b/docs/reference/api/index.rst index c1c9ce0b2..01c7d7fe7 100644 --- a/docs/reference/api/index.rst +++ b/docs/reference/api/index.rst @@ -8,7 +8,7 @@ TODO: Explanation of each component .. toctree:: config - data + data/index fields field_components/index models diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index 8bcd21bbe..7e2ebedeb 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -149,10 +149,11 @@ class DataManager(nn.Module): Usage: To get data, use the next_train and next_eval functions. This data manager's next_train and next_eval methods will return 2 things: - 1. A Raybundle: This will contain the rays we are sampling, with latents and - conditionals attached (everything needed at inference) - 2. A "batch" of auxiliary information: This will contain the mask, the ground truth - pixels, etc needed to actually train, score, etc the model + + 1. A Raybundle: This will contain the rays we are sampling, with latents and + conditionals attached (everything needed at inference) + 2. A "batch" of auxiliary information: This will contain the mask, the ground truth + pixels, etc needed to actually train, score, etc the model Rationale: Because of this abstraction we've added, we can support more NeRF paradigms beyond the diff --git a/nerfstudio/data/dataparsers/nerfosr_dataparser.py b/nerfstudio/data/dataparsers/nerfosr_dataparser.py index 820a28956..f7b84b261 100644 --- a/nerfstudio/data/dataparsers/nerfosr_dataparser.py +++ b/nerfstudio/data/dataparsers/nerfosr_dataparser.py @@ -42,7 +42,7 @@ from nerfstudio.data.scene_box import SceneBox CONSOLE = Console(width=120) -def find_files(directory: str, exts: List[str]): +def _find_files(directory: str, exts: List[str]): """Find all files in a directory that have a certain file extension. Args: @@ -94,8 +94,8 @@ def get_camera_params( split_dir = f"{scene_dir}/{split}" # camera parameters files - intrinsics_files = find_files(f"{split_dir}/intrinsics", exts=["*.txt"]) - pose_files = find_files(f"{split_dir}/pose", exts=["*.txt"]) + intrinsics_files = _find_files(f"{split_dir}/intrinsics", exts=["*.txt"]) + pose_files = _find_files(f"{split_dir}/pose", exts=["*.txt"]) num_cams = len(pose_files) @@ -219,12 +219,12 @@ class NeRFOSR(DataParser): ) # --- images --- - image_filenames = find_files(f"{split_dir}/rgb", exts=["*.png", "*.jpg", "*.JPG", "*.PNG"]) + image_filenames = _find_files(f"{split_dir}/rgb", exts=["*.png", "*.jpg", "*.JPG", "*.PNG"]) # --- masks --- mask_filenames = [] if self.config.use_masks: - mask_filenames = find_files(f"{split_dir}/mask", exts=["*.png", "*.jpg", "*.JPG", "*.PNG"]) + mask_filenames = _find_files(f"{split_dir}/mask", exts=["*.png", "*.jpg", "*.JPG", "*.PNG"]) dataparser_outputs = DataparserOutputs( image_filenames=image_filenames, diff --git a/nerfstudio/data/dataparsers/scannet_dataparser.py b/nerfstudio/data/dataparsers/scannet_dataparser.py index d5a787799..5d2702f18 100644 --- a/nerfstudio/data/dataparsers/scannet_dataparser.py +++ b/nerfstudio/data/dataparsers/scannet_dataparser.py @@ -39,10 +39,14 @@ class ScanNetDataParserConfig(DataParserConfig): ScanNet dataset (https://www.scan-net.org/) is a large-scale 3D dataset of indoor scenes. This dataparser assumes that the dense stream was extracted from .sens files. Expected structure of scene directory: - - color/ - - depth/ - - intrinsic/ - - pose/ + + .. code-block:: text + + root/ + ├── color/ + ├── depth/ + ├── intrinsic/ + ├── pose/ """ _target: Type = field(default_factory=lambda: ScanNet) From 835a0151f28d45424dbab6d0cf507d67cebf428b Mon Sep 17 00:00:00 2001 From: Justin Kerr Date: Wed, 5 Apr 2023 07:43:39 -0700 Subject: [PATCH 4/5] Create nerfacto-big config for nerfacto with sampling, network size, resolution, and train steps cranked up (#1668) * Increase max_res of first proposal network * nerfacto-big params * skip nerfacto_big in test --------- Co-authored-by: Matt Tancik Co-authored-by: Ethan Weber --- nerfstudio/configs/method_configs.py | 40 ++++++++++++++++++++++++++++ tests/test_train.py | 1 + 2 files changed, 41 insertions(+) diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index 0bc56f6da..0357d0f9b 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -112,6 +112,46 @@ method_configs["nerfacto"] = TrainerConfig( viewer=ViewerConfig(num_rays_per_chunk=1 << 15), vis="viewer", ) +method_configs["nerfacto-big"] = TrainerConfig( + method_name="nerfacto", + steps_per_eval_batch=500, + steps_per_save=2000, + max_num_iterations=100000, + mixed_precision=True, + pipeline=VanillaPipelineConfig( + datamanager=VanillaDataManagerConfig( + dataparser=NerfstudioDataParserConfig(), + train_num_rays_per_batch=4096, + eval_num_rays_per_batch=4096, + camera_optimizer=CameraOptimizerConfig( + mode="SO3xR3", optimizer=RAdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-3) + ), + ), + model=NerfactoModelConfig( + eval_num_rays_per_chunk=1 << 15, + num_nerf_samples_per_ray=128, + num_proposal_samples_per_ray=(512, 256), + hidden_dim=128, + hidden_dim_color=128, + hidden_dim_transient=128, + max_res=3000, + proposal_weights_anneal_max_num_iters=5000, + log2_hashmap_size=21, + ), + ), + optimizers={ + "proposal_networks": { + "optimizer": RAdamOptimizerConfig(lr=1e-2, eps=1e-15), + "scheduler": None, + }, + "fields": { + "optimizer": RAdamOptimizerConfig(lr=1e-2, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=100000), + }, + }, + viewer=ViewerConfig(num_rays_per_chunk=1 << 15), + vis="viewer", +) method_configs["depth-nerfacto"] = TrainerConfig( method_name="depth-nerfacto", diff --git a/tests/test_train.py b/tests/test_train.py index b38e84205..f795f6060 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -21,6 +21,7 @@ BLACKLIST = [ "instant-ngp", "instant-ngp-bounded", "nerfacto", + "nerfacto-big", "volinga", "phototourism", "depth-nerfacto", From 2e74462c324519e950fa8faa7f034b3dc8b9900e Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Wed, 5 Apr 2023 10:30:01 -0700 Subject: [PATCH 5/5] Update nerfstudio data defaults (#1701) --- nerfstudio/data/dataparsers/nerfstudio_dataparser.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py index cffc09be1..7b283d747 100644 --- a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py +++ b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py @@ -46,7 +46,7 @@ class NerfstudioDataParserConfig(DataParserConfig): _target: Type = field(default_factory=lambda: Nerfstudio) """target class to instantiate""" - data: Path = Path("data/nerfstudio/poster") + data: Path = Path() """Directory or explicit json file path specifying location of data.""" scale_factor: float = 1.0 """How much to scale the camera origins by.""" @@ -76,6 +76,8 @@ class Nerfstudio(DataParser): def _generate_dataparser_outputs(self, split="train"): # pylint: disable=too-many-statements + assert self.config.data.exists(), f"Data directory {self.config.data} does not exist." + if self.config.data.suffix == ".json": meta = load_from_json(self.config.data) data_dir = self.config.data.parent