mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-22 15:45:39 +08:00
Merge branch 'main' of https://github.com/nerfstudio-project/nerfstudio
This commit is contained in:
commit
1b1bb224cf
24 files changed
+438
-148
No files matched your search
@@ -171,6 +171,7 @@ events.out.*
|
||||
# Data
|
||||
data
|
||||
!*/data
|
||||
!docs/reference/api/data/
|
||||
|
||||
# Misc
|
||||
old/
|
||||
|
||||
@@ -196,16 +196,17 @@ ns-export pointcloud --help
|
||||
|
||||
Using an existing dataset is great, but likely you want to use your own data! We support various methods for using your own data. Before it can be used in nerfstudio, the camera location and orientations must be determined and then converted into our format using `ns-process-data`. We rely on external tools for this, instructions and information can be found in the documentation.
|
||||
|
||||
| Data | Capture Device | Requirements | `ns-process-data` Speed |
|
||||
| ------------------------------------------------------------------------------------------------- | -------------- | ----------------------------------------------- | ----------------------- |
|
||||
| 📷 [Images](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#images-and-video) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐢 |
|
||||
| 📹 [Video](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#images-and-video) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐢 |
|
||||
| 🌎 [360 Data](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#360_data) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐢 |
|
||||
| 📱 [Polycam](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#polycam-capture) | IOS with LiDAR | [Polycam App](https://poly.cam/) | 🐇 |
|
||||
| 📱 [KIRI Engine](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#kiri-capture) | IOS or Android | [KIRI Engine App](https://www.kiriengine.com/) | 🐇 |
|
||||
| 📱 [Record3D](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#record3d-capture) | IOS with LiDAR | [Record3D app](https://record3d.app/) | 🐇 |
|
||||
| 🖥 [Metashape](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#metashape) | Any | [Metashape](https://www.agisoft.com/) | 🐇 |
|
||||
| 🛠 [Custom](https://docs.nerf.studio/en/latest/quickstart/data_conventions.html) | Any | Camera Poses | 🐇 |
|
||||
| Data | Capture Device | Requirements | `ns-process-data` Speed |
|
||||
| ---------------------------------------------------------------------------------------------------- | -------------- | ----------------------------------------------------------------- | ----------------------- |
|
||||
| 📷 [Images](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#images-and-video) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐢 |
|
||||
| 📹 [Video](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#images-and-video) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐢 |
|
||||
| 🌎 [360 Data](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#360_data) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐢 |
|
||||
| 📱 [Polycam](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#polycam-capture) | IOS with LiDAR | [Polycam App](https://poly.cam/) | 🐇 |
|
||||
| 📱 [KIRI Engine](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#kiri-capture) | IOS or Android | [KIRI Engine App](https://www.kiriengine.com/) | 🐇 |
|
||||
| 📱 [Record3D](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#record3d-capture) | IOS with LiDAR | [Record3D app](https://record3d.app/) | 🐇 |
|
||||
| 🖥 [Metashape](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#metashape) | Any | [Metashape](https://www.agisoft.com/) | 🐇 |
|
||||
| 🖥 [RealityCapture](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#realitycapture) | Any | [RealityCapture](https://www.capturingreality.com/realitycapture) | 🐇 |
|
||||
| 🛠 [Custom](https://docs.nerf.studio/en/latest/quickstart/data_conventions.html) | Any | Camera Poses | 🐇 |
|
||||
|
||||
## 5. Advanced Options
|
||||
|
||||
|
||||
@@ -42,6 +42,7 @@ extensions = [
|
||||
"sphinx_design",
|
||||
"sphinx.ext.mathjax",
|
||||
"sphinxext.opengraph",
|
||||
"sphinx.ext.viewcode",
|
||||
]
|
||||
|
||||
# List of patterns, relative to source directory, that match files and
|
||||
|
||||
@@ -118,33 +118,3 @@ Often times, you just want to play with the parameters of an existing model with
|
||||
# NOTE: the dataparser and associated configurations go at the end of the command
|
||||
ns-train {METHOD_NAME} --vis viewer {DATA_PARSER} --scale-factor 0.5
|
||||
```
|
||||
|
||||
### Extending Nerfstudio with custom methods
|
||||
In order to extend the Nerfstudio and register your own methods, you can package your code as a python package
|
||||
and register it with Nerfstudio as a `nerfstudio.method_configs` entrypoint in the `pyproject.toml` file.
|
||||
The Nerfstudio will automatically look for all registered methods and will register them to be used
|
||||
by method such as `ns-train`.
|
||||
|
||||
Here is an example:
|
||||
```python
|
||||
"""my_package/my_config.py"""
|
||||
|
||||
from nerfstudio.engine.trainer import TrainerConfig
|
||||
from nerfstudio.plugins.types import MethodSpecification
|
||||
|
||||
MyMethod = MethodSpecification(
|
||||
config=TrainerConfig(
|
||||
method_name="my-method",
|
||||
...
|
||||
),
|
||||
description="Custom description"
|
||||
)
|
||||
|
||||
"""pyproject.toml"""
|
||||
[project]
|
||||
name = "my_package"
|
||||
...
|
||||
|
||||
[project.entry-points.'nerfstudio.method_configs']
|
||||
my-method = 'my_package.my_config:MyMethod'
|
||||
```
|
||||
@@ -0,0 +1,75 @@
|
||||
# Adding a New Method
|
||||
|
||||
Nerfstudio aims to offer researchers a codebase that they can utilize to extend and develop novel methods. Our vision is for users to establish a distinct repository that imports nerfstudio and overrides pipeline components to cater to specific functionality requirements of the new approach. If any of the new features require modifications to the core nerfstudio repository and can be generally useful, we encourage you to submit a PR to enable others to benefit from it.
|
||||
|
||||
Examples are often the best way to learn, take a look at the [LERF](https://github.com/kerrj/lerf) repository for good example of how to use nerfstudio in your projects.
|
||||
|
||||
## File Structure
|
||||
|
||||
We recommend the following file structure:
|
||||
|
||||
```
|
||||
├── my_method
|
||||
│ ├── my_config.py
|
||||
│ ├── custom_pipeline.py [optional]
|
||||
│ ├── custom_model.py [optional]
|
||||
│ ├── custom_field.py [optional]
|
||||
│ ├── custom_datamanger.py [optional]
|
||||
│ ├── ...
|
||||
├── pyproject.toml
|
||||
```
|
||||
|
||||
## Registering custom model with nerfstudio
|
||||
|
||||
In order to extend the Nerfstudio and register your own methods, you can package your code as a python package
|
||||
and register it with Nerfstudio as a `nerfstudio.method_configs` entrypoint in the `pyproject.toml` file.
|
||||
Nerfstudio will automatically look for all registered methods and will register them to be used
|
||||
by methods such as `ns-train`.
|
||||
|
||||
First create a config file:
|
||||
|
||||
```python
|
||||
"""my_method/my_config.py"""
|
||||
|
||||
from nerfstudio.engine.trainer import TrainerConfig
|
||||
from nerfstudio.plugins.types import MethodSpecification
|
||||
|
||||
MyMethod = MethodSpecification(
|
||||
config=TrainerConfig(
|
||||
method_name="my-method",
|
||||
pipeline=...
|
||||
...
|
||||
),
|
||||
description="Custom description"
|
||||
)
|
||||
```
|
||||
|
||||
Then create a `pyproject.toml` file. This is where the entrypoint to your method is set and also where you can specify additional dependencies required by your codebase.
|
||||
|
||||
```python
|
||||
"""pyproject.toml"""
|
||||
|
||||
[project]
|
||||
name = "my_method"
|
||||
|
||||
dependencies = [
|
||||
"nerfstudio" # you may want to consider pinning the version, ie "nerfstudio==0.1.19"
|
||||
]
|
||||
|
||||
[project.entry-points.'nerfstudio.method_configs']
|
||||
my-method = 'my_method.my_config:MyMethod'
|
||||
```
|
||||
|
||||
finally run the following to register the method,
|
||||
|
||||
```
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
## Running custom method
|
||||
|
||||
After registering your method you should be able to run the method with,
|
||||
|
||||
```
|
||||
ns-train my-method --data DATA_DIR
|
||||
```
|
||||
@@ -30,8 +30,16 @@ class DataparserOutputs:
|
||||
"""Color of dataset background."""
|
||||
scene_box: SceneBox = SceneBox()
|
||||
"""Scene box of dataset. Used to bound the scene or provide the scene scale depending on model."""
|
||||
semantics: Optional[Semantics] = None
|
||||
"""Semantics information."""
|
||||
mask_filenames: Optional[List[Path]] = None
|
||||
"""Filenames for any masks that are required"""
|
||||
metadata: Dict[str, Any] = to_immutable_dict({})
|
||||
"""Dictionary of any metadata that be required for the given experiment.
|
||||
Will be processed by the InputDataset to create any additional tensors that may be required.
|
||||
"""
|
||||
dataparser_transform: TensorType[3, 4] = torch.eye(4)[:3, :]
|
||||
"""Transform applied by the dataparser."""
|
||||
dataparser_scale: float = 1.0
|
||||
"""Scale applied by the dataparser."""
|
||||
|
||||
@dataclass
|
||||
class DataParser:
|
||||
@@ -115,48 +123,12 @@ You can also pull out information from the DataParserOutputs for other DataManga
|
||||
ray_generator = RayGenerator(dataparser_outputs.cameras)
|
||||
```
|
||||
|
||||
## Our Implementations
|
||||
## Included DataParsers
|
||||
|
||||
Below we enumerate the various dataparsers that we have implemented in our codebase. Feel free to use ours or add your own. Also any contributions are welcome and appreciated!
|
||||
```{toctree}
|
||||
---
|
||||
maxdepth: 2
|
||||
---
|
||||
|
||||
###### Nerfstudio
|
||||
|
||||
This is our custom dataparser. We have a script to convert images or videos with COLMAP to this format.
|
||||
|
||||
```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/nerfstudio_dataparser.py
|
||||
:color: primary
|
||||
:outline:
|
||||
See the code!
|
||||
```
|
||||
|
||||
###### Blender
|
||||
|
||||
We support the synthetic Blender dataset from the original NeRF paper.
|
||||
|
||||
```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/blender_dataparser.py
|
||||
:color: primary
|
||||
:outline:
|
||||
See the code!
|
||||
```
|
||||
|
||||
###### Instant NGP
|
||||
|
||||
This supports the Instant NGP dataset.
|
||||
|
||||
```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/instant_ngp_dataparser.py
|
||||
:color: primary
|
||||
:outline:
|
||||
See the code!
|
||||
```
|
||||
|
||||
###### Record3D
|
||||
|
||||
This dataparser can use recorded data from a >= iPhone 12 Pro using the [Record3D app](https://record3d.app/) . Record a video and export with the `EXR + JPG sequence` format. Unzip export and `rgb` folder before training.
|
||||
|
||||
For more information on capturing with Record3D, see the [Custom Dataset Docs](/quickstart/custom_dataset.md).
|
||||
|
||||
```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/record3d_dataparser.py
|
||||
:color: primary
|
||||
:outline:
|
||||
See the code!
|
||||
../../reference/api/data/dataparsers
|
||||
```
|
||||
@@ -75,6 +75,7 @@ nerfology/model_components/index
|
||||
:hidden:
|
||||
:caption: Developer Guides
|
||||
|
||||
developer_guides/new_methods
|
||||
developer_guides/pipelines/index
|
||||
developer_guides/viewer/viewer_overview
|
||||
developer_guides/config
|
||||
|
||||
@@ -20,6 +20,7 @@ We Currently support the following custom data types:
|
||||
| 📱 [KIRI Engine](kiri) | IOS or Android | [KIRI Engine App](https://www.kiriengine.com/) | 🐇 |
|
||||
| 📱 [Record3D](record3d) | IOS with LiDAR | [Record3D app](https://record3d.app/) | 🐇 |
|
||||
| 🖥 [Metashape](metashape) | Any | [Metashape](https://www.agisoft.com/) | 🐇 |
|
||||
| 🖥 [RealityCapture](realitycapture) | Any | [RealityCapture](https://www.capturingreality.com/realitycapture) | 🐇 |
|
||||
|
||||
(images_and_video)=
|
||||
|
||||
@@ -286,6 +287,26 @@ ns-process-data metashape --data {data directory} --xml {xml file} --output-dir
|
||||
ns-train nerfacto --data {output directory}
|
||||
```
|
||||
|
||||
(realitycapture)=
|
||||
|
||||
## RealityCapture
|
||||
|
||||
1. Align your images using RealityCapture. `ALIGNMENT -> Align Images`
|
||||
|
||||
2. Export the camera alignment as a `csv` file. Choose `Internal/External camera parameters`
|
||||
|
||||
3. Convert the data to the nerfstudio format.
|
||||
|
||||
```bash
|
||||
ns-process-data realitycapture --data {data directory} --csv {csv file} --output-dir {output directory}
|
||||
```
|
||||
|
||||
4. Train with nerfstudio!
|
||||
|
||||
```bash
|
||||
ns-train nerfacto --data {output directory}
|
||||
```
|
||||
|
||||
(360_data)=
|
||||
|
||||
## 360 Data (Equirectangular)
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
.. _dataset:
|
||||
|
||||
Data
|
||||
============
|
||||
|
||||
Data Parsers
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.dataparsers
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
Datasets
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.datasets
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
Dataloaders
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.utils.dataloaders
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
Pixel Samplers
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.pixel_samplers
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
Scene Box
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.scene_box
|
||||
:members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,31 @@
|
||||
.. _datamanagers:
|
||||
|
||||
Datamanagers
|
||||
============
|
||||
|
||||
Base
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.datamanagers.base_datamanager
|
||||
:members:
|
||||
|
||||
Depth Datamanager
|
||||
-----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.datamanagers.depth_datamanager
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
SDF Datamanager
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.datamanagers.sdf_datamanager
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
Semantic Datamanager
|
||||
---------------------
|
||||
|
||||
.. automodule:: nerfstudio.data.datamanagers.semantic_datamanager
|
||||
:members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,117 @@
|
||||
.. _dataparser:
|
||||
|
||||
Data Parsers
|
||||
============
|
||||
|
||||
* `Base Data Parser`_
|
||||
* `ARKitScenes`_
|
||||
* `Blender`_
|
||||
* `D-NeRF`_
|
||||
* `dycheck`_
|
||||
* `Instant-NGP`_
|
||||
* `Minimal`_
|
||||
* `NeRF-OSR`_
|
||||
* `Nerfstudio`_
|
||||
* `nuScenes`_
|
||||
* `Phototourism`_
|
||||
* `ScanNet`_
|
||||
* `SDFStudio`_
|
||||
* `sitcoms3D`_
|
||||
|
||||
|
||||
Base Data Parser
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.dataparsers.base_dataparser
|
||||
:members:
|
||||
|
||||
ARKitScenes
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.dataparsers.arkitscenes_dataparser
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
Blender
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.dataparsers.blender_dataparser
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
D-NeRF
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.dataparsers.dnerf_dataparser
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
dycheck
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.dataparsers.dycheck_dataparser
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
Instant-NGP
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.dataparsers.instant_ngp_dataparser
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
Minimal
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.dataparsers.minimal_dataparser
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
NeRF-OSR
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.dataparsers.nerfosr_dataparser
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
Nerfstudio
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.dataparsers.nerfstudio_dataparser
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
nuScenes
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.dataparsers.nuscenes_dataparser
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
Phototourism
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.dataparsers.phototourism_dataparser
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
ScanNet
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.dataparsers.scannet_dataparser
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
SDFStudio
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.dataparsers.sdfstudio_dataparser
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
sitcoms3D
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.dataparsers.sitcoms3d_dataparser
|
||||
:members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,24 @@
|
||||
.. _datasets:
|
||||
|
||||
Datasets
|
||||
============
|
||||
|
||||
Base
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.datasets.base_dataset
|
||||
:members:
|
||||
|
||||
SDF Dataset
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.datasets.sdf_dataset
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
Semantic Dataset
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.datasets.semantic_dataset
|
||||
:members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,26 @@
|
||||
.. _dataset:
|
||||
|
||||
Data
|
||||
============
|
||||
|
||||
.. toctree::
|
||||
:titlesonly:
|
||||
|
||||
dataparsers
|
||||
datamanagers
|
||||
datasets
|
||||
utils
|
||||
|
||||
Pixel Samplers
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.pixel_samplers
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
Scene Box
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.scene_box
|
||||
:members:
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,28 @@
|
||||
.. _datautils:
|
||||
|
||||
Utils
|
||||
============
|
||||
|
||||
Base
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.utils.colmap_parsing_utils
|
||||
:members:
|
||||
|
||||
Data
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.utils.data_utils
|
||||
:members:
|
||||
|
||||
Dataloader
|
||||
----------------
|
||||
|
||||
.. automodule:: nerfstudio.data.utils.dataloaders
|
||||
:members:
|
||||
|
||||
Nerfstudio Collate
|
||||
-------------------
|
||||
|
||||
.. automodule:: nerfstudio.data.utils.nerfstudio_collate
|
||||
:members:
|
||||
@@ -8,7 +8,7 @@ TODO: Explanation of each component
|
||||
.. toctree::
|
||||
|
||||
config
|
||||
data
|
||||
data/index
|
||||
fields
|
||||
field_components/index
|
||||
models
|
||||
|
||||
+11
-10
@@ -6,21 +6,22 @@ We provide a command line interface for training your own NeRFs (no coding neces
|
||||
|
||||
Here are the popular commands that we offer. If you've cloned the repo, you can also look at the [pyproject.toml file](https://github.com/nerfstudio-project/nerfstudio/blob/main/pyproject.toml) at the `[project.scripts]` section for details.
|
||||
|
||||
| Command | Description | Filename |
|
||||
| ---------------- | -------------------------------------- | ---------------------------------- |
|
||||
| ns-install-cli | Install tab completion for all scripts | scripts/completions/install.py |
|
||||
| ns-process-data | Generate a dataset from your own data | scripts/process_data.py |
|
||||
| ns-download-data | Download existing captures | scripts/downloads/download_data.py |
|
||||
| ns-train | Generate a NeRF | scripts/train.py |
|
||||
| ns-viewer | View a trained NeRF | scripts/viewer/run_viewer.py |
|
||||
| ns-eval | Run evaluation metrics for your Model | scripts/eval.py |
|
||||
| ns-render | Render out a video of your NeRF | scripts/render.py |
|
||||
| ns-export | Export a NeRF into other formats | scripts/exporter.py |
|
||||
| Command | Description | Filename |
|
||||
| ------------------------------------ | -------------------------------------- | ---------------------------------- |
|
||||
| [ns-install-cli](ns_install_cli) | Install tab completion for all scripts | scripts/completions/install.py |
|
||||
| [ns-process-data](ns_process_data) | Generate a dataset from your own data | scripts/process_data.py |
|
||||
| [ns-download-data](ns_download_data) | Download existing captures | scripts/downloads/download_data.py |
|
||||
| [ns-train](ns_train) | Generate a NeRF | scripts/train.py |
|
||||
| [ns-viewer](ns_viewer) | View a trained NeRF | scripts/viewer/run_viewer.py |
|
||||
| [ns-eval](ns_eval) | Run evaluation metrics for your Model | scripts/eval.py |
|
||||
| [ns-render](ns_render) | Render out a video of your NeRF | scripts/render.py |
|
||||
| [ns-export](ns_export) | Export a NeRF into other formats | scripts/exporter.py |
|
||||
|
||||
```{toctree}
|
||||
:maxdepth: 1
|
||||
:hidden:
|
||||
|
||||
ns_install_cli
|
||||
ns_process_data
|
||||
ns_download_data
|
||||
ns_train
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
# ns-install-cli
|
||||
|
||||
```{eval-rst}
|
||||
.. argparse::
|
||||
:module: scripts.completions.install
|
||||
:func: get_parser_fn
|
||||
:prog: ns-install-cli
|
||||
:nodefault:
|
||||
```
|
||||
@@ -112,6 +112,46 @@ method_configs["nerfacto"] = TrainerConfig(
|
||||
viewer=ViewerConfig(num_rays_per_chunk=1 << 15),
|
||||
vis="viewer",
|
||||
)
|
||||
method_configs["nerfacto-big"] = TrainerConfig(
|
||||
method_name="nerfacto",
|
||||
steps_per_eval_batch=500,
|
||||
steps_per_save=2000,
|
||||
max_num_iterations=100000,
|
||||
mixed_precision=True,
|
||||
pipeline=VanillaPipelineConfig(
|
||||
datamanager=VanillaDataManagerConfig(
|
||||
dataparser=NerfstudioDataParserConfig(),
|
||||
train_num_rays_per_batch=4096,
|
||||
eval_num_rays_per_batch=4096,
|
||||
camera_optimizer=CameraOptimizerConfig(
|
||||
mode="SO3xR3", optimizer=RAdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-3)
|
||||
),
|
||||
),
|
||||
model=NerfactoModelConfig(
|
||||
eval_num_rays_per_chunk=1 << 15,
|
||||
num_nerf_samples_per_ray=128,
|
||||
num_proposal_samples_per_ray=(512, 256),
|
||||
hidden_dim=128,
|
||||
hidden_dim_color=128,
|
||||
hidden_dim_transient=128,
|
||||
max_res=3000,
|
||||
proposal_weights_anneal_max_num_iters=5000,
|
||||
log2_hashmap_size=21,
|
||||
),
|
||||
),
|
||||
optimizers={
|
||||
"proposal_networks": {
|
||||
"optimizer": RAdamOptimizerConfig(lr=1e-2, eps=1e-15),
|
||||
"scheduler": None,
|
||||
},
|
||||
"fields": {
|
||||
"optimizer": RAdamOptimizerConfig(lr=1e-2, eps=1e-15),
|
||||
"scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=100000),
|
||||
},
|
||||
},
|
||||
viewer=ViewerConfig(num_rays_per_chunk=1 << 15),
|
||||
vis="viewer",
|
||||
)
|
||||
|
||||
method_configs["depth-nerfacto"] = TrainerConfig(
|
||||
method_name="depth-nerfacto",
|
||||
|
||||
@@ -149,10 +149,11 @@ class DataManager(nn.Module):
|
||||
Usage:
|
||||
To get data, use the next_train and next_eval functions.
|
||||
This data manager's next_train and next_eval methods will return 2 things:
|
||||
1. A Raybundle: This will contain the rays we are sampling, with latents and
|
||||
conditionals attached (everything needed at inference)
|
||||
2. A "batch" of auxiliary information: This will contain the mask, the ground truth
|
||||
pixels, etc needed to actually train, score, etc the model
|
||||
|
||||
1. A Raybundle: This will contain the rays we are sampling, with latents and
|
||||
conditionals attached (everything needed at inference)
|
||||
2. A "batch" of auxiliary information: This will contain the mask, the ground truth
|
||||
pixels, etc needed to actually train, score, etc the model
|
||||
|
||||
Rationale:
|
||||
Because of this abstraction we've added, we can support more NeRF paradigms beyond the
|
||||
|
||||
@@ -42,7 +42,7 @@ from nerfstudio.data.scene_box import SceneBox
|
||||
CONSOLE = Console(width=120)
|
||||
|
||||
|
||||
def find_files(directory: str, exts: List[str]):
|
||||
def _find_files(directory: str, exts: List[str]):
|
||||
"""Find all files in a directory that have a certain file extension.
|
||||
|
||||
Args:
|
||||
@@ -94,8 +94,8 @@ def get_camera_params(
|
||||
split_dir = f"{scene_dir}/{split}"
|
||||
|
||||
# camera parameters files
|
||||
intrinsics_files = find_files(f"{split_dir}/intrinsics", exts=["*.txt"])
|
||||
pose_files = find_files(f"{split_dir}/pose", exts=["*.txt"])
|
||||
intrinsics_files = _find_files(f"{split_dir}/intrinsics", exts=["*.txt"])
|
||||
pose_files = _find_files(f"{split_dir}/pose", exts=["*.txt"])
|
||||
|
||||
num_cams = len(pose_files)
|
||||
|
||||
@@ -219,12 +219,12 @@ class NeRFOSR(DataParser):
|
||||
)
|
||||
|
||||
# --- images ---
|
||||
image_filenames = find_files(f"{split_dir}/rgb", exts=["*.png", "*.jpg", "*.JPG", "*.PNG"])
|
||||
image_filenames = _find_files(f"{split_dir}/rgb", exts=["*.png", "*.jpg", "*.JPG", "*.PNG"])
|
||||
|
||||
# --- masks ---
|
||||
mask_filenames = []
|
||||
if self.config.use_masks:
|
||||
mask_filenames = find_files(f"{split_dir}/mask", exts=["*.png", "*.jpg", "*.JPG", "*.PNG"])
|
||||
mask_filenames = _find_files(f"{split_dir}/mask", exts=["*.png", "*.jpg", "*.JPG", "*.PNG"])
|
||||
|
||||
dataparser_outputs = DataparserOutputs(
|
||||
image_filenames=image_filenames,
|
||||
|
||||
@@ -46,7 +46,7 @@ class NerfstudioDataParserConfig(DataParserConfig):
|
||||
|
||||
_target: Type = field(default_factory=lambda: Nerfstudio)
|
||||
"""target class to instantiate"""
|
||||
data: Path = Path("data/nerfstudio/poster")
|
||||
data: Path = Path()
|
||||
"""Directory or explicit json file path specifying location of data."""
|
||||
scale_factor: float = 1.0
|
||||
"""How much to scale the camera origins by."""
|
||||
@@ -76,6 +76,8 @@ class Nerfstudio(DataParser):
|
||||
def _generate_dataparser_outputs(self, split="train"):
|
||||
# pylint: disable=too-many-statements
|
||||
|
||||
assert self.config.data.exists(), f"Data directory {self.config.data} does not exist."
|
||||
|
||||
if self.config.data.suffix == ".json":
|
||||
meta = load_from_json(self.config.data)
|
||||
data_dir = self.config.data.parent
|
||||
|
||||
@@ -39,10 +39,14 @@ class ScanNetDataParserConfig(DataParserConfig):
|
||||
ScanNet dataset (https://www.scan-net.org/) is a large-scale 3D dataset of indoor scenes.
|
||||
This dataparser assumes that the dense stream was extracted from .sens files.
|
||||
Expected structure of scene directory:
|
||||
- color/
|
||||
- depth/
|
||||
- intrinsic/
|
||||
- pose/
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
root/
|
||||
├── color/
|
||||
├── depth/
|
||||
├── intrinsic/
|
||||
├── pose/
|
||||
"""
|
||||
|
||||
_target: Type = field(default_factory=lambda: ScanNet)
|
||||
|
||||
@@ -399,3 +399,6 @@ def entrypoint():
|
||||
|
||||
if __name__ == "__main__":
|
||||
entrypoint()
|
||||
|
||||
# For sphinx docs
|
||||
get_parser_fn = lambda: tyro.extras.get_parser(main) # noqa
|
||||
@@ -21,6 +21,7 @@ BLACKLIST = [
|
||||
"instant-ngp",
|
||||
"instant-ngp-bounded",
|
||||
"nerfacto",
|
||||
"nerfacto-big",
|
||||
"volinga",
|
||||
"phototourism",
|
||||
"depth-nerfacto",
|
||||
|
||||
Reference in new issue
Block a user