diff --git a/.gitignore b/.gitignore
index ab8a4e108..cfa4358d2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -171,6 +171,7 @@ events.out.*
# Data
data
!*/data
+!docs/reference/api/data/
# Misc
old/
diff --git a/.vscode/launch.json b/.vscode/launch.json
index 118282928..da293a07b 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -112,5 +112,42 @@
"--include_mono_prior=True"
]
},
+ {
+ "name": "Python: NeuS-facto on Replica",
+ "type": "python",
+ "request": "launch",
+ "program": "scripts/train.py",
+ "console": "integratedTerminal",
+ "justMyCode": false,
+ "env": {"CUDA_VISIBLE_DEVICES":"0"},
+ "args": [
+ "neus-facto",
+ "--pipeline.model.sdf-field.inside-outside=True",
+ "--pipeline.model.mono-depth-loss-mult=0.1",
+ "--pipeline.model.mono-normal-loss-mult=0.05",
+ "--vis=wandb",
+ "sdfstudio-data",
+ "--data=data/sdfstudio-demo-data/replica-room0",
+ "--include_mono_prior=True"
+ ]
+ },
+ {
+ "name": "Python: NeuS-facto on DTU (no mono-prior)",
+ "type": "python",
+ "request": "launch",
+ "program": "scripts/train.py",
+ "console": "integratedTerminal",
+ "justMyCode": false,
+ "env": {"CUDA_VISIBLE_DEVICES":"0"},
+ "args": [
+ "neus-facto",
+ "--pipeline.model.sdf-field.inside-outside=False",
+ "--vis=wandb",
+ "sdfstudio-data",
+ "--data=data/sdfstudio-demo-data/dtu-scan65",
+ "--include_mono_prior=False",
+ "--auto-orient=True"
+ ]
+ },
]
}
diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 000000000..90f5ae01c
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,126 @@
+{
+ "files.watcherExclude": {
+ "**/__pycache__/": true,
+ "**/.ipynb_checkpoints": true,
+ "**/*.ipynb": true,
+ "**/old*": true,
+ "**/runs*": true,
+ "**/temp*": true
+ },
+ "editor.defaultFormatter": "esbenp.prettier-vscode",
+ "[javascript]": {
+ "editor.defaultFormatter": "esbenp.prettier-vscode"
+ },
+ "[javascriptreact]": {
+ "editor.defaultFormatter": "esbenp.prettier-vscode"
+ },
+ "[typescript]": {
+ "editor.defaultFormatter": "esbenp.prettier-vscode"
+ },
+ "[typescriptreact]": {
+ "editor.defaultFormatter": "esbenp.prettier-vscode"
+ },
+ "prettier.configPath": ".prettierrc.js",
+ "typescript.suggestionActions.enabled": false,
+ "javascript.suggestionActions.enabled": false,
+ "[python]": {
+ "editor.defaultFormatter": "ms-python.python",
+ "editor.codeActionsOnSave": {
+ "source.organizeImports": true
+ }
+ },
+ "editor.formatOnSave": true,
+ "python.envFile": "${workspaceFolder}/.env",
+ "python.formatting.provider": "black",
+ "python.formatting.blackArgs": ["--line-length=120"],
+ "python.linting.pylintEnabled": true,
+ "python.linting.flake8Enabled": false,
+ "python.linting.enabled": true,
+ "python.sortImports.args": ["--src=${workspaceFolder}", "--profile", "black"],
+ "python.testing.unittestEnabled": false,
+ "python.testing.pytestEnabled": true,
+ "python.linting.ignorePatterns": [
+ "**/site-packages/**/*.py",
+ ".vscode/*.py",
+ "external/**/*.py"
+ ],
+ "esbonio.sphinx.confDir": "",
+ // eslint extension options
+ "javascript.validate.enable": false,
+ "eslint.enable": true,
+ "eslint.validate": [
+ "javascript",
+ "javascriptreact",
+ "typescript",
+ "typescriptreact"
+ ],
+ "C_Cpp.errorSquiggles": "Disabled",
+ "files.associations": {
+ "array": "cpp",
+ "bitset": "cpp",
+ "string_view": "cpp",
+ "initializer_list": "cpp",
+ "utility": "cpp",
+ "__hash_table": "cpp",
+ "__split_buffer": "cpp",
+ "deque": "cpp",
+ "iterator": "cpp",
+ "queue": "cpp",
+ "stack": "cpp",
+ "string": "cpp",
+ "unordered_map": "cpp",
+ "vector": "cpp",
+ "atomic": "cpp",
+ "*.tcc": "cpp",
+ "cctype": "cpp",
+ "chrono": "cpp",
+ "clocale": "cpp",
+ "cmath": "cpp",
+ "codecvt": "cpp",
+ "condition_variable": "cpp",
+ "cstdarg": "cpp",
+ "cstddef": "cpp",
+ "cstdint": "cpp",
+ "cstdio": "cpp",
+ "cstdlib": "cpp",
+ "cstring": "cpp",
+ "ctime": "cpp",
+ "cwchar": "cpp",
+ "cwctype": "cpp",
+ "exception": "cpp",
+ "algorithm": "cpp",
+ "filesystem": "cpp",
+ "functional": "cpp",
+ "memory": "cpp",
+ "memory_resource": "cpp",
+ "optional": "cpp",
+ "ratio": "cpp",
+ "system_error": "cpp",
+ "tuple": "cpp",
+ "type_traits": "cpp",
+ "fstream": "cpp",
+ "iomanip": "cpp",
+ "iosfwd": "cpp",
+ "iostream": "cpp",
+ "istream": "cpp",
+ "limits": "cpp",
+ "mutex": "cpp",
+ "new": "cpp",
+ "ostream": "cpp",
+ "sstream": "cpp",
+ "stdexcept": "cpp",
+ "streambuf": "cpp",
+ "thread": "cpp",
+ "typeinfo": "cpp",
+ "__nullptr": "cpp",
+ "__config": "cpp",
+ "__locale": "cpp",
+ "__bit_reference": "cpp",
+ "ios": "cpp",
+ "__atomic": "cpp",
+ "__node_handle": "cpp"
+ },
+ "python.analysis.typeCheckingMode": "basic",
+ "python.analysis.diagnosticMode": "workspace",
+ "eslint.packageManager": "yarn"
+}
diff --git a/docs/conf.py b/docs/conf.py
index 2071057b9..824a61d78 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -42,6 +42,7 @@ extensions = [
"sphinx_design",
"sphinx.ext.mathjax",
"sphinxext.opengraph",
+ "sphinx.ext.viewcode",
]
# List of patterns, relative to source directory, that match files and
diff --git a/docs/developer_guides/new_methods.md b/docs/developer_guides/new_methods.md
index c0484ca9e..661629d15 100644
--- a/docs/developer_guides/new_methods.md
+++ b/docs/developer_guides/new_methods.md
@@ -73,3 +73,7 @@ After registering your method you should be able to run the method with,
```
ns-train my-method --data DATA_DIR
```
+
+## Adding to the _nerf.studio_ documentation
+
+We invite researchers to contribute their own methods to our online documentation. You can find more information on how to do this {ref}`here`.
diff --git a/docs/developer_guides/pipelines/dataparsers.md b/docs/developer_guides/pipelines/dataparsers.md
index 3a38664f9..19b964e5a 100644
--- a/docs/developer_guides/pipelines/dataparsers.md
+++ b/docs/developer_guides/pipelines/dataparsers.md
@@ -30,8 +30,16 @@ class DataparserOutputs:
"""Color of dataset background."""
scene_box: SceneBox = SceneBox()
"""Scene box of dataset. Used to bound the scene or provide the scene scale depending on model."""
- semantics: Optional[Semantics] = None
- """Semantics information."""
+ mask_filenames: Optional[List[Path]] = None
+ """Filenames for any masks that are required"""
+ metadata: Dict[str, Any] = to_immutable_dict({})
+ """Dictionary of any metadata that be required for the given experiment.
+ Will be processed by the InputDataset to create any additional tensors that may be required.
+ """
+ dataparser_transform: TensorType[3, 4] = torch.eye(4)[:3, :]
+ """Transform applied by the dataparser."""
+ dataparser_scale: float = 1.0
+ """Scale applied by the dataparser."""
@dataclass
class DataParser:
@@ -115,48 +123,12 @@ You can also pull out information from the DataParserOutputs for other DataManga
ray_generator = RayGenerator(dataparser_outputs.cameras)
```
-## Our Implementations
+## Included DataParsers
-Below we enumerate the various dataparsers that we have implemented in our codebase. Feel free to use ours or add your own. Also any contributions are welcome and appreciated!
+```{toctree}
+---
+maxdepth: 2
+---
-###### Nerfstudio
-
-This is our custom dataparser. We have a script to convert images or videos with COLMAP to this format.
-
-```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/nerfstudio_dataparser.py
-:color: primary
-:outline:
-See the code!
-```
-
-###### Blender
-
-We support the synthetic Blender dataset from the original NeRF paper.
-
-```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/blender_dataparser.py
-:color: primary
-:outline:
-See the code!
-```
-
-###### Instant NGP
-
-This supports the Instant NGP dataset.
-
-```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/instant_ngp_dataparser.py
-:color: primary
-:outline:
-See the code!
-```
-
-###### Record3D
-
-This dataparser can use recorded data from a >= iPhone 12 Pro using the [Record3D app](https://record3d.app/) . Record a video and export with the `EXR + JPG sequence` format. Unzip export and `rgb` folder before training.
-
-For more information on capturing with Record3D, see the [Custom Dataset Docs](/quickstart/custom_dataset.md).
-
-```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/record3d_dataparser.py
-:color: primary
-:outline:
-See the code!
+../../reference/api/data/dataparsers
```
diff --git a/docs/developer_guides/viewer/custom_gui.md b/docs/developer_guides/viewer/custom_gui.md
new file mode 100644
index 000000000..09275dac1
--- /dev/null
+++ b/docs/developer_guides/viewer/custom_gui.md
@@ -0,0 +1,106 @@
+# Custom GUI
+
+We provide support for custom viewer GUI elements that can be defined in any `nn.Module`. Although we don't have any specific use cases in mind, here are some examples of what can be achieved with this feature:
+
+- Using text input to modify the rendering
+- Logging numerical values to the viewer
+- Using checkboxes to turn off and on losses
+- Using a dropdown to switch between appearances
+
+## Adding an Element
+
+To define a custom element, create an instance of one of the provided classes in `nerfstudio.viewer.server.viewer_elements`, and assign it as a class variable in your `nn.Module`.
+
+```python
+from nerfstudio.viewer.server.viewer_elements import ViewerNumber
+
+class MyClass(nn.Module):#must inherit from nn.Module
+ def __init__(self):
+ # Must be a class variable
+ self.custom_value = ViewerNumber(name="My Value", default_value=1.0)
+```
+**Element Hierarchy**
+The viewer recursively searches all `nn.Module` children of the base `Pipeline` object, and arranges parameters into folders based on their variable names.
+For example, a `ViewerElement` defined in `pipeline.model.field` will be in the "Custom/model/field" folder in the GUI.
+
+**Reading the value**
+To read the value of a custom element, simply access its `value` attribute. In this case it will be `1.0` unless modified by the user in the viewer.
+
+```python
+current_value = self.custom_value.value
+```
+
+**Callbacks**
+You can register a callback that will be called whenever a new value for your GUI element is available. For example, one can use a callback to update config parameters when elements are changed:
+```python
+def on_change_callback(handle: GuiHandle):
+ self.config.example_parameter = handle.get_value()
+
+self.custom_checkbox = ViewerCheckbox(
+ name="Checkbox",
+ default_value = False,
+ cb_hook = on_change_callback
+)
+```
+The GuiHandle value is a viser object which you can use to get the new value.
+
+**Thread safety**
+Note that `ViewerElement` values can change asynchronously to model execution. So, it's best practice to store the value of a viewer element once at the beginning
+of a forward pass and refer to the static variable afterwards.
+```python
+class MyModel(Model):
+ def __init__(self):
+ self.slider = ViewerSlider(name="Slider", default_value=0.5, min_value=0.0, max_value=1.0)
+
+ def get_outputs(self,ray)
+ slider_val = self.slider.value
+ #self.slider.value could change after this, unsafe to use
+
+```
+
+
+**Writing to the element**
+You can write to a viewer element in Python, which provides a convenient way to track values in your code without the need for wandb/tensorboard or relying on `print` statements.
+
+```python
+self.custom_value = x
+```
+
+:::{admonition} Warning
+:class: warning
+
+Note that updating module state while training can have unexpected side effects. It is up to the user to ensure that GUI actions are safe. In most cases, we recommend wrapping GUI related logic in `if self.training:` to ensure that the effects are only applied during inference.
+:::
+
+## Example Elements
+
+```{image} imgs/custom_controls.png
+:align: center
+:width: 400
+```
+
+This was created with the following elements:
+
+```python
+from nerfstudio.viewer.server.viewer_elements import *
+
+class MyModel(Model):
+ def __init__(self):
+ self.a = ViewerButton(name="My Button", cb_hook=self.handle_btn)
+ self.b = ViewerNumber(name="Number", default_value=1.0)
+ self.c = ViewerCheckbox(name="Checkbox", default_value=False)
+ self.d = ViewerDropdown(name="Dropdown", default_value="A", options=["A", "B"])
+ self.e = ViewerSlider(name="Slider", default_value=0.5, min_value=0.0, max_value=1.0)
+ self.f = ViewerText(name="Text", default_value="Hello World")
+ self.g = ViewerVec3(name="3D Vector", default_value=(0.1, 0.7, 0.1))
+
+ self.rgb_renderer = RGBRenderer()
+...
+class RGBRenderer(nn.Module):
+ def __init__(self):
+ #lives in "Custom/model/rgb_renderer" GUI folder
+ self.a = ViewerRGB(name="F", default_value=(0.1, 0.7, 0.1))
+...
+```
+
+For more information on the available classes and their arguments, refer to the [API documentation](../../reference/api/viewer.rst)
diff --git a/docs/developer_guides/viewer/imgs/custom_controls.png b/docs/developer_guides/viewer/imgs/custom_controls.png
new file mode 100644
index 000000000..557f76b5a
Binary files /dev/null and b/docs/developer_guides/viewer/imgs/custom_controls.png differ
diff --git a/docs/developer_guides/viewer/imgs/viewer_dark.png b/docs/developer_guides/viewer/imgs/viewer_dark.png
deleted file mode 100644
index d2abe9a10..000000000
Binary files a/docs/developer_guides/viewer/imgs/viewer_dark.png and /dev/null differ
diff --git a/docs/developer_guides/viewer/imgs/viewer_figure.png b/docs/developer_guides/viewer/imgs/viewer_figure.png
deleted file mode 100644
index 3d0fd2846..000000000
Binary files a/docs/developer_guides/viewer/imgs/viewer_figure.png and /dev/null differ
diff --git a/docs/developer_guides/viewer/imgs/viewer_light.png b/docs/developer_guides/viewer/imgs/viewer_light.png
deleted file mode 100644
index 2220e3a8c..000000000
Binary files a/docs/developer_guides/viewer/imgs/viewer_light.png and /dev/null differ
diff --git a/docs/developer_guides/viewer/index.md b/docs/developer_guides/viewer/index.md
new file mode 100644
index 000000000..eebc29d32
--- /dev/null
+++ b/docs/developer_guides/viewer/index.md
@@ -0,0 +1,22 @@
+# Viewer
+
+> We have a real-time web viewer that requires no installation. It's available at [https://viewer.nerf.studio/](https://viewer.nerf.studio/), where you can connect to your training job.
+
+The viewer is built on [Viser](https://github.com/brentyi/viser/tree/main/viser) using [ThreeJS](https://threejs.org/) and packaged into a [ReactJS](https://reactjs.org/) application. This client viewer application will connect via a websocket to a server running on your machine.
+
+```{toctree}
+:titlesonly:
+
+custom_gui
+local_viewer
+```
+
+## Acknowledgements and references
+
+We thank the authors and contributors to the following repos, which we've started, used, and modified for our use-cases.
+
+- [Viser](https://github.com/brentyi/viser/tree/main/viser) - made by [Brent Yi](https://github.com/brentyi)
+- [meshcat-python](https://github.com/rdeits/meshcat-python) - made by [Robin Deits](https://github.com/rdeits)
+- [meshcat](https://github.com/rdeits/meshcat) - made by [Robin Deits](https://github.com/rdeits)
+- [ThreeJS](https://threejs.org/)
+- [ReactJS](https://reactjs.org/)
diff --git a/docs/developer_guides/viewer/local_viewer.md b/docs/developer_guides/viewer/local_viewer.md
new file mode 100644
index 000000000..d5912bd89
--- /dev/null
+++ b/docs/developer_guides/viewer/local_viewer.md
@@ -0,0 +1,62 @@
+# Local Server
+
+If you are unable to connect to `https://viewer.nerf.studio`, want to use Safari, or want develop the viewer codebase, you can launch your own local viewer.
+
+## Installing Dependencies
+
+```shell
+cd nerfstudio/viewer/app
+```
+
+Install npm (to install yarn) and yarn
+
+```shell
+sudo apt-get install npm
+npm install --global yarn
+```
+
+Install nvm and set the node version
+Install nvm with [instructions](https://heynode.com/tutorial/install-nodejs-locally-nvm/).
+
+```shell
+nvm install 17.8.0
+```
+
+Now running `node --version` in the shell should print "v17.8.0".
+Install package.json dependencies and start the client viewer app:
+
+```shell
+yarn install
+```
+
+## Launch the web client
+
+From the `nerfstudio/viewer/app` folder, run:
+
+```shell
+yarn start
+```
+
+The local webserver runs on port 4000 by default,
+so when `ns-train` is running, you can connect to the viewer locally at
+[http://localhost:4000/?websocket_url=ws://localhost:7007](http://localhost:4000/?websocket_url=ws://localhost:7007)
+
+## FAQ
+
+### Engine node incompatible
+
+While running `yarn install`, you run into: `The engine "node" is incompatible with this module.`
+
+**Solution**:
+
+Install nvm with instructions at [instructions](https://heynode.com/tutorial/install-nodejs-locally-nvm/).
+
+```shell
+nvm install 17.8.0
+```
+
+If you cannot install nvm, try ignoring the engines
+
+```shell
+yarn install --ignore-engines
+```
diff --git a/docs/developer_guides/viewer/viewer_overview.md b/docs/developer_guides/viewer/viewer_overview.md
deleted file mode 100644
index 55dbab2f2..000000000
--- a/docs/developer_guides/viewer/viewer_overview.md
+++ /dev/null
@@ -1,98 +0,0 @@
-# Viewer overview
-
-> We have a real-time web viewer that requires no installation. It's available at [https://viewer.nerf.studio/](https://viewer.nerf.studio/), where you can connect to your training job. If you want to understand how the viewer works and contribute to it's features, this section is for you!
-
-The viewer is built using [ThreeJS](https://threejs.org/) and packaged into a [ReactJS](https://reactjs.org/) application. This client viewer application will connect via a websocket to a server running on your machine. The following figure helps to illustrate how our viewer framework works:
-
-```{image} imgs/viewer_light.png
-:align: center
-:class: only-light
-```
-
-```{image} imgs/viewer_dark.png
-:align: center
-:class: only-dark
-```
-
-## System design
-
-#### Connection between nerfstudio and the Bridge Server
-
-In the center, we have the Bridge Server, which facilitates the connection between nerfstudio code and the Client App. This server runs on the same machine that you are using nerfstudio. It has a TCP Request/Reply (REQ/REP) connection that nerfstudio can connect to with the Viewer object (left). We use [ZeroMQ](https://zeromq.org/), an open-sourced messaging library, to implement this lightweight TCP connection. The Viewer class can send commands to the Bridge Server and receive replies. The Bridge Server will either dispatch commands to the Client App via a websocket or it will return information stored in the Bridge Server state.
-
-#### Connection between the Bridge Server and the Client App
-
-The connection between the Bridge Server and the Client App works with WebSockets.
-
-- **WebSocket connection** - The WebSocket is used by the Bridge Server to dispatch commands coming from the nerfstudio TCP connection. Commands can be used for drawing primitives, for setting the transform of objects, for the setting various properties, and more.
-
-## Installing and running locally
-
-#### Running the Client App
-
-```shell
-cd nerfstudio/viewer/app
-```
-
-Install npm (to install yarn) and yarn
-
-```shell
-sudo apt-get install npm
-npm install --global yarn
-```
-
-Install nvm and set the node version
-Install nvm with [instructions](https://heynode.com/tutorial/install-nodejs-locally-nvm/).
-
-```shell
-nvm install 17.8.0
-```
-
-Now running `node --version` in the shell should print "v17.8.0".
-Install package.json dependencies and start the client viewer app
-
-```shell
-yarn install
-yarn start
-```
-
-The local webserver runs on port 4000 by default,
-so when `ns-train` is running, you can connect to the viewer locally at
-[http://localhost:4000/?websocket_url=ws://localhost:7007](http://localhost:4000/?websocket_url=ws://localhost:7007)
-
-#### Running the Bridge Server Manually
-
-The viewer bridge server runs on the same machine that you use for training. The training code will connect to this server with a lightweight TCP connection using ZMQ. The training job will launch the viewer bridge server is you specify `--viewer.launch-bridge-server` in the terminal. Otherwise, you can launch the bridge server manually with the following script.
-
-```python
-ns-bridge-server --help
-```
-
-## Acknowledgements and references
-
-We thank the authors and contributors to the following repos, which we've started, used, and modified for our use-cases.
-
-- [meshcat-python](https://github.com/rdeits/meshcat-python) - made by [Robin Deits](https://github.com/rdeits)
-- [meshcat](https://github.com/rdeits/meshcat) - made by [Robin Deits](https://github.com/rdeits)
-- [ThreeJS](https://threejs.org/)
-- [ReactJS](https://reactjs.org/)
-
-## FAQ
-
-#### Engine node incompatible
-
-While running `yarn install`, you run into: `The engine "node" is incompatible with this module.`
-
-**Solution**:
-
-Install nvm with instructions at [instructions](https://heynode.com/tutorial/install-nodejs-locally-nvm/).
-
-```shell
-nvm install 17.8.0
-```
-
-If you cannot install nvm, try ignoring the engines
-
-```
-yarn install --ignore-engines
-```
diff --git a/docs/extensions/blender_addon.md b/docs/extensions/blender_addon.md
index 2f410ff4f..842d9bb38 100644
--- a/docs/extensions/blender_addon.md
+++ b/docs/extensions/blender_addon.md
@@ -1,58 +1,75 @@
# Blender VFX add-on
-
+
-
## Overview
-This Blender add-on allows for compositing with a Nerfstudio render as a background layer by generating a camera path JSON file from the Blender camera path, as well as a way to import Nerfstudio JSON files as a Blender camera baked with the Nerfstudio camera path. This is achieved by importing a mesh or point-cloud representation of the NeRF scene from Nerfstudio into Blender and getting the camera coordinates relative to the transformations of the NeRF representation. Dynamic FOV from the Blender camera is supported and will match the Nerfstudio render. Perspective and equirectangular cameras are also supported.
+This Blender add-on allows for compositing with a Nerfstudio render as a background layer by generating a camera path JSON file from the Blender camera path, as well as a way to import Nerfstudio JSON files as a Blender camera baked with the Nerfstudio camera path. This add-on also allows compositing multiple NeRF objects into a NeRF scene. This is achieved by importing a mesh or point-cloud representation of the NeRF scene from Nerfstudio into Blender and getting the camera coordinates relative to the transformations of the NeRF representation. Dynamic FOV from the Blender camera is supported and will match the Nerfstudio render. Perspective and equirectangular cameras are supported.
-## Prerequisites
-
-First install [Blender](https://www.blender.org/). The python file add-on can be downloaded from the main repo in the `scripts/blender` folder. Before starting, export the mesh or point cloud representation of the NeRF from Nerfstudio, which will be used as reference for the actual NeRF in the Blender scene. Mesh export at a good quality is preferred, however, if the export is not clear or the NeRF is very large, a detailed point cloud export will also work.
+
+
+
## Add-on Setup Instructions
-1. Open a new or existing scene to add the NeRF background in.
+1. The add-on requires Blender 3.0 or newer, install Blender [here](https://www.blender.org/).
+2. Download Blender Add-on Script
-2. Install and enable the add-on in Blender in Edit → Preferences → Add-Ons. The add-on will be visible in the Render Properties tab on the right panel.
-
-
-
-
+3. Install and enable the add-on in Blender in `Edit → Preferences → Add-Ons`. The add-on will be visible in the Render Properties tab on the right panel.
+
+
+
-3. Import the mesh or point cloud representation of the NeRF into the scene. It is preferred that a mesh is used at a good enough fidelity. You may need to crop the mesh further. Since it is used as a reference and won't be visible in the final render, only the parts that the blender animation will interact with may be necessary to import.
+4. The add-on should now be installed in the `Render Properties` panel
+
+
+
+
+## Scene Setup
+
+1. Export the mesh or point cloud representation of the NeRF from Nerfstudio, which will be used as reference for the actual NeRF in the Blender scene. Mesh export at a good quality is preferred, however, if the export is not clear or the NeRF is large, a detailed point cloud export will also work.
+
+2. Import the mesh or point cloud representation of the NeRF into the scene. You may need to crop the mesh further. Since it is used as a reference and won't be visible in the final render, only the parts that the blender animation will interact with may be necessary to import.
+
+3. Select the NeRF mesh or point cloud in the add-on.
4. Resize, position, or rotate the NeRF representation to fit your scene.
-
## Generate Nerfstudio JSON Camera Path from Blender Camera
1. There are a few ways to hide the reference mesh for the Blender render
- - In object properties, select "Shadow Catcher". This makes the representation invisible in the render, but all shadows cast on it will render. You may have to switch to the cycles renderer to see the shadow catcher option.
-
-
- - Note: This may not give ideal results if the mesh is not very clear or occludes other objects in the scene. If this is the case, you can hide the mesh from the render instead by clicking the camera button in the Outliner next to its name.
-
+ - In object properties, select "Shadow Catcher". This makes the representation invisible in the render, but all shadows cast on it will render. You may have to switch to the cycles renderer to see the shadow catcher option.
+
+
+
+
+ - Note: This may not give ideal results if the mesh is not very clear or occludes other objects in the scene. If this is the case, you can hide the mesh from the render instead by clicking the camera button in the Outliner next to its name.
+
+
+
2. Verify that the animation plays and the NeRF representation does not occlude the camera.
-3. Go to the Nerfstudio Add-on in Render Properties and expand the "Nerfstudio Path Generator" tab in the panel. Use the object selector to select the NeRF representation. Then, select the file path for the output JSON camera path file.
-4. Click "Generate JSON file". The output JSON file is named `camera_path_blender.json`.
+3. Go to the Nerfstudio Add-on in Render Properties and expand the "Nerfstudio Path Generator" tab in the panel. Use the object selector to select the NeRF representation. Then, select the file path for the output JSON camera path file.
+4. Click "Generate JSON file". The output JSON file is named `camera_path_blender.json`.
+
-
+
5. Render the NeRF with the generated camera path using Nerfstudio in the command prompt or terminal.
-6. Before rendering the Blender animation, go to the Render Properties and in the Film settings select "Transparent" so that the render will be rendered with a clear background to allow it to be composited over the NeRF render.
+6. Before rendering the Blender animation, go to the Render Properties and in the Film settings select "Transparent" so that the render will be rendered with a clear background to allow it to be composited over the NeRF render.
+
+
-
-7. Now the scene can be rendered and composited over the camera aligned Nerfstudio render.
+7. Now the scene can be rendered and composited over the camera aligned Nerfstudio render.
+
+
### Examples
@@ -63,23 +80,29 @@ First install [Blender](https://www.blender.org/). The python file add-on can be
### Additional details
-- You can also apply an environment texture to the Blender scene by using Nerfstudio to render an equirectangular image (360 image) of the NeRF from a place in the scene.
-- The settings for the Blender equirectangular camera are: "Panoramic" camera type and panorama type of "Equirectangular".
-
+- You can also apply an environment texture to the Blender scene by using Nerfstudio to render an equirectangular image (360 image) of the NeRF from a place in the scene.
+- The settings for the Blender equirectangular camera are: "Panoramic" camera type and panorama type of "Equirectangular".
+
+
+
- The generated JSON camera path follows the user specified frame start, end, and step fields in the Output Properties in Blender. The JSON file specifies the user specified x and y render resolutions at the given % in the Output Properties.
- The add-on computes the camera coordinates based on the active camera in the scene.
- FOV animated changes of the camera will be matched with the NeRF render.
- Perspective and equirectangular cameras are supported and can be selected within Blender.
- The generated JSON file can be imported into Nerfstudio. Each keyframe of the camera transform in the frame sequence in Blender will be a keyframe in Nerfstudio. The exported JSON camera path is baked, where each frame in the sequence is a keyframe. This is to ensure that frame interpolation across Nerfstudio and Blender do not differ.
+
+
- It is recommended to run the camera path in the Nerfstudio web interface with the NeRF to ensure that the NeRF is visible throughout the camera path.
- The fps and keyframe timestamps are based on the "Frame Rate" setting in the Output Properties in Blender.
- The NeRF representation can also be transformed (position, rotation, and scale) as well as animated.
+- The pivot point of the NeRF representation should not be changed
- It is recommended to export a high fidelity mesh as the NeRF representation from Nerfstudio. However if that is not possible or the scene is too large, a point cloud representation also works.
- For compositing, it is recommended to convert the video render into image frames to ensure the Blender and NeRF renders are in sync.
- Currently, dynamic camera focus is not supported.
- Fisheye and orthographic cameras are not supported.
+- A walkthrough of this section is included in the tutorial video.
## Create Blender Camera from Nerfstudio JSON Camera Path
@@ -87,7 +110,6 @@ First install [Blender](https://www.blender.org/). The python file add-on can be
-
1. Expand the "Nerfstudio Camera Generator" tab in the panel. After inputting the NeRF representation, select the JSON Nerfstudio file and click "Create Camera from JSON"
2. A new camera named "NerfstudioCamera" should appear in the viewport with the camera path and FOV of the input file. This camera's type will match the Nerfstudio input file, except fisheye cameras.
@@ -103,13 +125,93 @@ First install [Blender](https://www.blender.org/). The python file add-on can be
- Scale of the camera is not keyframed from the Nerfstudio camera path.
- This newly created camera has a sensor fit of "Vertical"
+## Compositing NeRF Objects in NeRF Environments
+
+You can composite NeRF objects into a scene with a NeRF background by rendering the cropped NeRF object along with an accumulation render as an alpha mask and compositing that over the background NeRF render.
+
+
+
+
+
+1. Import the background NeRF scene as a point cloud (or mesh, but point cloud is preferred for large scenes).
+
+2. Export a cropped NeRF mesh of the NeRF object(s)
+ - Open the NeRF object from the Nerfstudio viewer and select "Crop Viewport" and accordingly adjust the Scale and Center values of the bounding box to crop the NeRF scene to around the bounds of the object of interest.
+
+
+
+
+
+ - Copy over the scale and center values into the Export panel and export the NeRF object as a mesh (point cloud will also work but shadows can be rendered if the object is a mesh)
+
+ - Keep note of the scale and center values for the crop. This can be done by creating a new JSON camera path in the editor which will add a crop section towards the end of the file.
+
+
+
+
+
+
+3. Import the NeRF object representation. Rescale and position the scene and NeRF object and background environment. You can also animate either of them.
+
+4. (Optional) To add shadows of the NeRF object(s)
+ - Add a plane representing the ground of the environment. In object properties, select "Shadow Catcher" under Visibility. You may have to switch to the cycles renderer to see the shadow catcher option.
+
+ - In the object properties of the NeRF object, go to the Ray Visibility section and deselect the "Camera" option. This will hide the mesh in the Blender render, but keep its shadow.
+
+
+
+
+
+
+ - Render the Blender animation with only the shadow of the object on the shadow catcher visible. Render with a transparent background by selecting "Transparent" under the Film Settings in the Render Properties.
+
+5. Go to the Nerfstudio Add-on in Render Properties and expand the "Nerfstudio Path Generator" tab in the panel. Use the object selector to select the NeRF environment representation. Then, select the file path for the output JSON camera path file. Click "Generate JSON file". The output JSON file is named `camera_path_blender.json`. It is recommended to rename this JSON file to keep track of the multiple camera paths that will be used to construct the full render.
+
+6. Generate a Nerfstudio camera path now for the NeRF object. Select the object selector to select the NeRF object. Before generating the new JSON file, you may need to rename the previously generated camera path for the environment or move it to a different directory otherwise the new JSON file will overwrite it. Click "Generate JSON file". The new output JSON file is named `camera_path_blender.json`. You will need to repeat this process for each NeRF object you want to composite in your scene.
+
+7. Render the NeRF of the background environment using the generated camera path for the environment using the Nerfstudio in the command line or terminal.
+
+8. Render the NeRF object
+ - Open the recently generated blender Nerfstudio JSON camera path and add the crop parameters to the camera path. This section will be placed towards the end of the Blender Nerfstudio camera path after the `is_cycle` field. This can be done by copying over the crop section from the JSON camera path with the scene crop which was exported earlier. Alternatively, you can enter the crop values manually in this format.
+
+
+
+
+ - Render the NeRF object using Nerfstudio and the edited camera path in the command line or terminal. This will be the RGB render.
+
+ - Next, render the accumulation render as an alpha mask of the NeRF object by adding the command line argument `--rendered-output-names accumulation` to the render command.
+
+
+
+
+
+9. Convert each of the Nerfstudio render videos into an image sequence of frames as PNG or JPG files. This will ensure that the frames will be aligned when compositing. You can convert the video mp4 to an image sequence by creating a Blender Video Editing file and rendering the mp4 as JPGs or PNGs.
+
+10. Composite the NeRF renders in a video editing software such as Adobe Premiere Pro.
+ - Place the render of the Nerfstudio background environment at the lowest layer, then place the shadow render of the NeRF object if created.
+
+ - Place the RGB NeRF render of the NeRF object over the environment (and shadow if present) layers and then place the accumulation NeRF object render over the RGB NeRF object render.
+
+ - Apply a filter to use the accumulation render as an alpha mask. In Premiere Pro, apply the effect "Track Matte Key" to the RGB render and select the "Matte" as the video track of the accumulation render and under "Composite Using" select "Matte Luma".
+
+### Additional Details
+- The RGB and accumulation renders will need to be rendered for each NeRF cropped object in the scene, but not for the NeRF environment if it is not cropped.
+- If you will composite a shadow layer, the quality of the exported mesh of the NeRF object should have enough fidelity to cast a shadow, but the texture doesn't need to be clear.
+- If motion tracking or compositing over real camera footage, you can add planes or cubes to represent walls or doorways as shadow catcher or holdout objects. This will composite the shadow layer over the NeRF environment and help create alpha masks.
+- The pivot point of the NeRF representations should not be changed.
+- A walkthrough of this section is included in the tutorial video.
+
+### Examples
+
+
+
+
## Implementation Details
-For generating the JSON camera path, we iterate over the scene frame sequence (from the start to the end with step intervals) and get the camera 4x4 world matrix at each frame. The world transformation matrix gives the position, rotation, and scale of the camera. We then obtain the world matrix of the NeRF representation at each frame and transform the camera coordinates with this to get the final camera world matrix. This allows us to re-position, rotate, and scale the NeRF representation in Blender and generate the right camera path to render the NeRF accordingly in Nerfstudio. Additionally, we calculate the FOV of the camera at each frame based on the sensor fit (horizontal or vertical), angle of view, and aspect ratio.
+For generating the JSON camera path, we iterate over the scene frame sequence (from the start to the end with step intervals) and get the camera 4x4 world matrix at each frame. The world transformation matrix gives the position, rotation, and scale of the camera. We then obtain the world matrix of the NeRF representation at each frame and transform the camera coordinates with this to get the final camera world matrix. This allows us to re-position, rotate, and scale the NeRF representation in Blender and generate the right camera path to render the NeRF accordingly in Nerfstudio. Additionally, we calculate the FOV of the camera at each frame based on the sensor fit (horizontal or vertical), angle of view, and aspect ratio.
Next, we construct the list of keyframes which is very similar to the world matrices of the transformed camera matrix.
Camera properties in the JSON file are based on user specified fields such as resolution (user specified in Output Properties in Blender), camera type (Perspective or Equirectangular). In the JSON file, `aspect` is specified as 1.0, `smoothness_value` is set to 0, and `is_cycle` is set to false. The Nerfstudio render is the fps specified in Blender where the duration is the total number of frames divided by the fps.
Finally, we construct the full JSON object and write it to the file path specified by the user.
-For generating the camera from the JSON file, we create a new Blender camera based on the input file and iterate through the `camera_path` field in the JSON to get the world matrix of the object from the `matrix_to_world` and similarly get the FOV from the `fov` fields. At each iteration, we set the camera to these parameters and insert a keyframe based on the position, rotation, and scale of the camera as well as the focal length of the camera based on the vertical FOV input.
-
+For generating the camera from the JSON file, we create a new Blender camera based on the input file and iterate through the `camera_path` field in the JSON to get the world matrix of the object from the `matrix_to_world` and similarly get the FOV from the `fov` fields. At each iteration, we set the camera to these parameters and insert a keyframe based on the position, rotation, and scale of the camera as well as the focal length of the camera based on the vertical FOV input.
diff --git a/docs/extensions/sdfstudio.md b/docs/extensions/sdfstudio.md
index 8f8134662..4c8f24874 100644
--- a/docs/extensions/sdfstudio.md
+++ b/docs/extensions/sdfstudio.md
@@ -34,6 +34,7 @@ We intend to integrate many of the SDFStudio improvments back into the nerstudio
Supported methods:
- NeuS
+- NeuS-facto
## Citation
diff --git a/docs/index.md b/docs/index.md
index d28393883..ee611c573 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -77,7 +77,7 @@ nerfology/model_components/index
developer_guides/new_methods
developer_guides/pipelines/index
-developer_guides/viewer/viewer_overview
+developer_guides/viewer/index
developer_guides/config
developer_guides/debugging_tools/index
```
diff --git a/docs/nerfology/methods/in2n.md b/docs/nerfology/methods/in2n.md
new file mode 100644
index 000000000..bd9ebccb6
--- /dev/null
+++ b/docs/nerfology/methods/in2n.md
@@ -0,0 +1,96 @@
+# Instruct-NeRF2NeRF
+
+
Editing 3D Scenes with Instructions
+
+```{button-link} https://instruct-nerf2nerf.github.io/
+:color: primary
+:outline:
+Paper Website
+```
+
+```{button-link} https://github.com/ayaanzhaque/instruct-nerf2nerf
+:color: primary
+:outline:
+Code
+```
+
+
+
+**Instruct-NeRF2NeRF enables instruction-based editing of NeRFs via a 2D diffusion model**
+
+## Installation
+
+First install nerfstudio dependencies. Then run:
+
+```bash
+pip install git+https://github.com/ayaanzhaque/instruct-nerf2nerf
+```
+
+## Running Instruct-NeRF2NeRF
+
+Details for running Instruct-NeRF2NeRF (built with Nerfstudio!) can be found [here](https://github.com/ayaanzhaque/instruct-nerf2nerf). Once installed, run:
+
+```bash
+ns-train in2n --help
+```
+
+Three variants of Instruct-NeRF2NeRF are provided:
+
+| Method | Description | Memory | Quality |
+| ------------ | ---------------------------- | ------ | ------- |
+| `in2n` | Full model, used in paper | ~15GB | Best |
+| `in2n-small` | Half precision model | ~12GB | Good |
+| `in2n-tiny` | Half prevision with no LPIPS | ~10GB | Ok |
+
+## Method
+
+### Overview
+
+Instruct-NeRF2NeRF is a method for editing NeRF scenes with text-instructions. Given a NeRF of a scene and the collection of images used to reconstruct it, the method uses an image-conditioned diffusion model ([InstructPix2Pix](https://www.timothybrooks.com/instruct-pix2pix)) to iteratively edit the input images while optimizing the underlying scene, resulting in an optimized 3D scene that respects the edit instruction. The paper demonstrates that their method is able to edit large-scale, real-world scenes, and is able to accomplish more realistic, targeted edits than prior work.
+
+## Pipeline
+
+
+
+This section will walk through each component of the Instruct-NeRF2NeRF method.
+
+### How it Works
+
+Instruct-NeRF2NeRF gradually updates a reconstructed NeRF scene by iteratively updating the dataset images while training the NeRF:
+
+1. An image is rendered from the scene at a training viewpoint.
+2. It is edited by InstructPix2Pix given a global text instruction.
+3. The training dataset image is replaced with the edited image.
+4. The NeRF continues training as usual.
+
+### Editing Images with InstructPix2Pix
+
+InstructPix2Pix is an image-editing diffusion model which can be prompted using text instructions. More details on InstructPix2Pix can be found [here](https://www.timothybrooks.com/instruct-pix2pix).
+
+Traditionally, at inference time, InstructPix2Pix takes as input random noise and is conditioned on an image (the image to edit) and a text instruction. The strength of the edit can be controlled using the image and text classifier-free guidance scales.
+
+To update a dataset image a given viewpoint, Instruct-NeRF2NeRF first takes the original, unedited training image as image conditioning and uses the global text instruction as text conditioning. The main input to the diffusion model is a noised version of the current render from the given viewpoint. The noise is sampled from a normal distribution and scaled based on a randomly chosen timestep. Then InstructPix2Pix slowly denoises the rendered image by predicting the noised version of the image at previous timesteps until the image is fully denoised. This will produce an edited version of the input image.
+
+This process mixes the information of the diffusion model, which attempts to edit the image, the current 3D structure of the NeRF, and view-consistent information from the unedited, ground-truth images. By combining this set of information, the edit is respected while maintaining 3D consistency.
+
+The code snippet for how an image is edited in the pipeline can be found [here](https://github.com/ayaanzhaque/instruct-nerf2nerf/blob/main/in2n/ip2p.py).
+
+### Iterative Dataset Update
+
+When NeRF training starts, the dataset consists of the original, unedited images used to train the original scene. These images are saved separately to use as conditioning for InstructPix2Pix. At each optimization step, some number of NeRF optimization steps are performed, and then some number of images (often just one) are updated. The images are randomly ordered prior to training and then at each step, the images are chosen in order to edit. Once an image has been edited, it is replaced in the dataset. Importantly, at each NeRF step, rays are sampled across the entire dataset, meaning there is a mixed source of supervision between edited images and unedited images. This allows for a gradual optimization that balances maintaining the 3D structure and consistency of the NeRF as well as performing the edit.
+
+At early iterations of this process, the edited images may be inconistent with one another, as InstructPix2Pix often doesn't perform consistent edits across viewpoints. However, over time, since images are edited using the current render of the NeRF, the edits begin to converge towards a globally consistent depiction of the underlying scene. Here is an example of how the underlying dataset evolves and becomes more consistent.
+
+
+
+The traditional method for supervising NeRFs using diffusion models is to use a Score Distillation Sampling (SDS) loss, as proposed in [DreamFusion](https://dreamfusion3d.github.io/). The Iterative Dataset Update method can be viewed as a variant of SDS, as instead of updating a discrete set of images at each step, the loss is a mix of rays from various viewpoints which are edited to varying degrees. The results show that this leads to higher quality performance and more stable optimization.
+
+## Results
+
+For results, view the [project page](https://instruct-nerf2nerf.github.io/)!
diff --git a/docs/nerfology/methods/index.md b/docs/nerfology/methods/index.md
index 9eb1af0a5..68912bb7b 100644
--- a/docs/nerfology/methods/index.md
+++ b/docs/nerfology/methods/index.md
@@ -1,8 +1,8 @@
# Methods
-We provide a set of pre implemented nerfstudio methods.
+We provide a set of pre implemented nerfstudio methods.
-**The goal of nerfstudio is to modularize the various NeRF techniques as much as possible.**
+**The goal of nerfstudio is to modularize the various NeRF techniques as much as possible.**
As a result, many of the techniques from these pre-implemented methods can be mixed 🎨.
@@ -20,9 +20,9 @@ To list the available methods run:
ns-train --help
```
-## Guides
+## Implemented Methods
-In addition to their implementations, we have provided guides that walk through each of these method.
+The following methods are supported in nerfstudio:
```{toctree}
:maxdepth: 1
@@ -30,5 +30,27 @@ In addition to their implementations, we have provided guides that walk through
Mip-NeRF
Nerfacto
Instant-NGP
+ Instruct-NeRF2NeRF
Semantic NeRF-W
```
+
+(own_method_docs)=
+
+## Adding Your Own Method
+
+If you're a researcher looking to develop new NeRF-related methods, we hope that you find nerfstudio to be a useful tool. We've provided documentation about integrating with the nerfstudio codebase, which you can find [here](../../developer_guides/new_methods.md).
+
+We also welcome additions to the list of methods above. To do this, simply create a pull request that adds a markdown file describing the model to the docs/nerfology/methods folder, and update the list in this file. For reference on the layout, you can check out the [Instruct-NeRF2NeRF](in2n) page. Please try to include the following information:
+
+- Installation instructions
+- Instructions for running the method
+- Requirements (dataset, GPU, ect)
+- Method description (the more detailed the better, treat it like a blog post)
+
+You are welcome to include assets (such as images or video) in the description, but please host them elsewhere.
+
+:::{admonition} Note
+:class: note
+
+Please ensure that the documentation is clear and easy to understand for other users who may want to try out your method.
+:::
diff --git a/docs/nerfology/methods/nerfacto.md b/docs/nerfology/methods/nerfacto.md
index a38f91bff..c1137884a 100644
--- a/docs/nerfology/methods/nerfacto.md
+++ b/docs/nerfology/methods/nerfacto.md
@@ -7,13 +7,21 @@
```bash
ns-train nerfacto --help
```
-
+
+We provide a few additional variants:
+
+| Method | Description | Memory | Speed |
+| ---------------- | --------------------- | ------ | ------ |
+| `nerfacto` | Default Model | ~6GB | Fast |
+| `nerfacto-big` | Larger higher quality | ~12GB | Slower |
+| `depth-nerfacto` | Supervise on depth | ~6GB | Fast |
+
## Method
-
+
### Overview
-
+
We created the nerfacto model to act as our default for real data captures of static scenes. The model is not existing published work, but rather a combination of many published methods that we have found work well for real data. This guide discusses the details of the model, understanding the [NeRF model](./nerf.md) is a prerequisite.
-
+
```{admonition} TLDR
We combine the following techniques in this model:
* Camera pose refinement
@@ -22,55 +30,55 @@ We combine the following techniques in this model:
* Scene contraction
* Hash encoding
```
-
+
```{warning}
🏗️ This guide is under construction 🏗️
```
-
+
## Pipeline
-
+
```{image} imgs/nerfacto/models_nerfacto_pipeline-light.png
:align: center
:class: only-light
```
-
+
```{image} imgs/nerfacto/models_nerfacto_pipeline-dark.png
:align: center
:class: only-dark
```
-
+
Here is an overview pipeline for nerfacto, we will walk through each component in this guide.
-
+
### Pose Refinement
-
+
It is not uncommon to have errors in the predicted camera poses. This is even more of a factor when using poses acquired from devices such as phones (ie. if you use the Record3D IOS app to capture data). Misaligned poses result in both cloudy artifacts in the scene and a reduction of sharpness and details. The NeRF framework allows us to backpropagate loss gradients to the input pose calculations. We can use this information to optimize and refine the poses.
-
+
### Piecewise Sampler
-
+
We use a Piecewise sampler to produce the initial set of samples of the scene. This sampler allocates half of the samples uniformly up to a distance of 1 from the camera. The remaining samples are distributed such that the step size increases with each sample. The step size is chosen such that the [frustums](../model_components/visualize_samples.ipynb) are scaled versions of themselves. By increasing the step sizes, we are able to sample distant objects will still having a dense set of samples for nearby objects.
-
+
### Proposal Sampler
-
+
The proposal sampler consolidates the sample locations to the regions of the scene that contribute most to the final render (typically the first surface intersection). This greatly improves reconstruction quality. The proposal network sampler requires a density function for the scene. The density function can be implemented in a variety of ways, we find that using a small fused-mlp with a hash encoding has sufficient accuracy and is fast. The proposal network sampler can be chained together with multiple density functions to further consolidate the sampling. We have found that two density functions are better than one. Larger than 2 leads to diminishing returns.
-
+
#### Density Field
-
+
The density field only needs to represent a coarse density representation of the scene to guide sampling. Combining a hash encoding with a small fused MLP (from [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn)) provides a fast way to query the scene. We can make it more efficient by decreasing the encoding dictionary size and number of feature levels. These simplifications have little impact on the reconstruction quality because the density function does not need to learn high frequency details during the initial passes.
-
+
### Nerfacto Field
-
+
```{image} imgs/nerfacto/models_nerfacto_field-light.png
:align: center
:class: only-light
:width: 600
```
-
+
```{image} imgs/nerfacto/models_nerfacto_field-dark.png
:align: center
:class: only-dark
:width: 600
```
-
+
```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/main/nerfstudio/models/nerfacto.py
:color: primary
:outline:
diff --git a/docs/reference/api/data.rst b/docs/reference/api/data.rst
deleted file mode 100644
index de7b57abe..000000000
--- a/docs/reference/api/data.rst
+++ /dev/null
@@ -1,39 +0,0 @@
-.. _dataset:
-
-Data
-============
-
-Data Parsers
-----------------
-
-.. automodule:: nerfstudio.data.dataparsers
- :members:
- :show-inheritance:
-
-Datasets
-----------------
-
-.. automodule:: nerfstudio.data.datasets
- :members:
- :show-inheritance:
-
-Dataloaders
-----------------
-
-.. automodule:: nerfstudio.data.utils.dataloaders
- :members:
- :show-inheritance:
-
-Pixel Samplers
-----------------
-
-.. automodule:: nerfstudio.data.pixel_samplers
- :members:
- :show-inheritance:
-
-Scene Box
-----------------
-
-.. automodule:: nerfstudio.data.scene_box
- :members:
- :show-inheritance:
\ No newline at end of file
diff --git a/docs/reference/api/data/datamanagers.rst b/docs/reference/api/data/datamanagers.rst
new file mode 100644
index 000000000..7779c9e10
--- /dev/null
+++ b/docs/reference/api/data/datamanagers.rst
@@ -0,0 +1,31 @@
+.. _datamanagers:
+
+Datamanagers
+============
+
+Base
+----------------
+
+.. automodule:: nerfstudio.data.datamanagers.base_datamanager
+ :members:
+
+Depth Datamanager
+-----------------
+
+.. automodule:: nerfstudio.data.datamanagers.depth_datamanager
+ :members:
+ :show-inheritance:
+
+SDF Datamanager
+----------------
+
+.. automodule:: nerfstudio.data.datamanagers.sdf_datamanager
+ :members:
+ :show-inheritance:
+
+Semantic Datamanager
+---------------------
+
+.. automodule:: nerfstudio.data.datamanagers.semantic_datamanager
+ :members:
+ :show-inheritance:
\ No newline at end of file
diff --git a/docs/reference/api/data/dataparsers.rst b/docs/reference/api/data/dataparsers.rst
new file mode 100644
index 000000000..e35f83ea0
--- /dev/null
+++ b/docs/reference/api/data/dataparsers.rst
@@ -0,0 +1,117 @@
+.. _dataparser:
+
+Data Parsers
+============
+
+* `Base Data Parser`_
+* `ARKitScenes`_
+* `Blender`_
+* `D-NeRF`_
+* `dycheck`_
+* `Instant-NGP`_
+* `Minimal`_
+* `NeRF-OSR`_
+* `Nerfstudio`_
+* `nuScenes`_
+* `Phototourism`_
+* `ScanNet`_
+* `SDFStudio`_
+* `sitcoms3D`_
+
+
+Base Data Parser
+----------------
+
+.. automodule:: nerfstudio.data.dataparsers.base_dataparser
+ :members:
+
+ARKitScenes
+----------------
+
+.. automodule:: nerfstudio.data.dataparsers.arkitscenes_dataparser
+ :members:
+ :show-inheritance:
+
+Blender
+----------------
+
+.. automodule:: nerfstudio.data.dataparsers.blender_dataparser
+ :members:
+ :show-inheritance:
+
+D-NeRF
+----------------
+
+.. automodule:: nerfstudio.data.dataparsers.dnerf_dataparser
+ :members:
+ :show-inheritance:
+
+dycheck
+----------------
+
+.. automodule:: nerfstudio.data.dataparsers.dycheck_dataparser
+ :members:
+ :show-inheritance:
+
+Instant-NGP
+----------------
+
+.. automodule:: nerfstudio.data.dataparsers.instant_ngp_dataparser
+ :members:
+ :show-inheritance:
+
+Minimal
+----------------
+
+.. automodule:: nerfstudio.data.dataparsers.minimal_dataparser
+ :members:
+ :show-inheritance:
+
+NeRF-OSR
+----------------
+
+.. automodule:: nerfstudio.data.dataparsers.nerfosr_dataparser
+ :members:
+ :show-inheritance:
+
+Nerfstudio
+----------------
+
+.. automodule:: nerfstudio.data.dataparsers.nerfstudio_dataparser
+ :members:
+ :show-inheritance:
+
+nuScenes
+----------------
+
+.. automodule:: nerfstudio.data.dataparsers.nuscenes_dataparser
+ :members:
+ :show-inheritance:
+
+Phototourism
+----------------
+
+.. automodule:: nerfstudio.data.dataparsers.phototourism_dataparser
+ :members:
+ :show-inheritance:
+
+ScanNet
+----------------
+
+.. automodule:: nerfstudio.data.dataparsers.scannet_dataparser
+ :members:
+ :show-inheritance:
+
+SDFStudio
+----------------
+
+.. automodule:: nerfstudio.data.dataparsers.sdfstudio_dataparser
+ :members:
+ :show-inheritance:
+
+sitcoms3D
+----------------
+
+.. automodule:: nerfstudio.data.dataparsers.sitcoms3d_dataparser
+ :members:
+ :show-inheritance:
diff --git a/docs/reference/api/data/datasets.rst b/docs/reference/api/data/datasets.rst
new file mode 100644
index 000000000..ee4fd1ab5
--- /dev/null
+++ b/docs/reference/api/data/datasets.rst
@@ -0,0 +1,24 @@
+.. _datasets:
+
+Datasets
+============
+
+Base
+----------------
+
+.. automodule:: nerfstudio.data.datasets.base_dataset
+ :members:
+
+SDF Dataset
+----------------
+
+.. automodule:: nerfstudio.data.datasets.sdf_dataset
+ :members:
+ :show-inheritance:
+
+Semantic Dataset
+----------------
+
+.. automodule:: nerfstudio.data.datasets.semantic_dataset
+ :members:
+ :show-inheritance:
\ No newline at end of file
diff --git a/docs/reference/api/data/index.rst b/docs/reference/api/data/index.rst
new file mode 100644
index 000000000..6e10f9384
--- /dev/null
+++ b/docs/reference/api/data/index.rst
@@ -0,0 +1,26 @@
+.. _dataset:
+
+Data
+============
+
+.. toctree::
+ :titlesonly:
+
+ dataparsers
+ datamanagers
+ datasets
+ utils
+
+Pixel Samplers
+----------------
+
+.. automodule:: nerfstudio.data.pixel_samplers
+ :members:
+ :show-inheritance:
+
+Scene Box
+----------------
+
+.. automodule:: nerfstudio.data.scene_box
+ :members:
+ :show-inheritance:
\ No newline at end of file
diff --git a/docs/reference/api/data/utils.rst b/docs/reference/api/data/utils.rst
new file mode 100644
index 000000000..5f225e6da
--- /dev/null
+++ b/docs/reference/api/data/utils.rst
@@ -0,0 +1,28 @@
+.. _datautils:
+
+Utils
+============
+
+Base
+----------------
+
+.. automodule:: nerfstudio.data.utils.colmap_parsing_utils
+ :members:
+
+Data
+----------------
+
+.. automodule:: nerfstudio.data.utils.data_utils
+ :members:
+
+Dataloader
+----------------
+
+.. automodule:: nerfstudio.data.utils.dataloaders
+ :members:
+
+Nerfstudio Collate
+-------------------
+
+.. automodule:: nerfstudio.data.utils.nerfstudio_collate
+ :members:
\ No newline at end of file
diff --git a/docs/reference/api/index.rst b/docs/reference/api/index.rst
index c1c9ce0b2..01c7d7fe7 100644
--- a/docs/reference/api/index.rst
+++ b/docs/reference/api/index.rst
@@ -8,7 +8,7 @@ TODO: Explanation of each component
.. toctree::
config
- data
+ data/index
fields
field_components/index
models
diff --git a/docs/reference/api/viewer.rst b/docs/reference/api/viewer.rst
index 508fddf68..4be105c86 100644
--- a/docs/reference/api/viewer.rst
+++ b/docs/reference/api/viewer.rst
@@ -3,6 +3,6 @@
Viewer
============
-.. automodule:: nerfstudio.viewer
+.. automodule:: nerfstudio.viewer.server.viewer_elements
:members:
:show-inheritance:
diff --git a/nerfstudio/cameras/camera_paths.py b/nerfstudio/cameras/camera_paths.py
index a313cad4d..9754a2bbf 100644
--- a/nerfstudio/cameras/camera_paths.py
+++ b/nerfstudio/cameras/camera_paths.py
@@ -125,10 +125,6 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras:
image_height = camera_path["render_height"]
image_width = camera_path["render_width"]
- if "camera_type" in camera_path:
- camera_type = camera_path["camera_type"]
- else:
- camera_type = "perspective"
if "camera_type" not in camera_path:
camera_type = CameraType.PERSPECTIVE
diff --git a/nerfstudio/cameras/camera_utils.py b/nerfstudio/cameras/camera_utils.py
index 79490ae67..8486f1097 100644
--- a/nerfstudio/cameras/camera_utils.py
+++ b/nerfstudio/cameras/camera_utils.py
@@ -164,8 +164,8 @@ def quaternion_matrix(quaternion: ArrayLike) -> np.ndarray:
def get_interpolated_poses(pose_a: ArrayLike, pose_b: ArrayLike, steps: int = 10) -> List[float]:
"""Return interpolation of poses with specified number of steps.
Args:
- poseA: first pose
- poseB: second pose
+ pose_a: first pose
+ pose_b: second pose
steps: number of steps the interpolated pose path should contain
"""
@@ -190,8 +190,8 @@ def get_interpolated_k(k_a, k_b, steps: int = 10) -> TensorType[3, 4]:
Returns interpolated path between two camera poses with specified number of steps.
Args:
- KA: camera matrix 1
- KB: camera matrix 2
+ k_a: camera matrix 1
+ k_b: camera matrix 2
steps: number of steps the interpolated pose path should contain
"""
Ks = []
diff --git a/nerfstudio/cameras/cameras.py b/nerfstudio/cameras/cameras.py
index e0ed08e4f..bda664642 100644
--- a/nerfstudio/cameras/cameras.py
+++ b/nerfstudio/cameras/cameras.py
@@ -62,11 +62,7 @@ CAMERA_MODEL_TO_TYPE = {
class Cameras(TensorDataclass):
"""Dataparser outputs for the image dataset and the ray generator.
- Note: currently only supports cameras with the same principal points and types. The reason we type
- the focal lengths, principal points, and image sizes as tensors is to allow for batched cameras
- down the line in cases where your batches of camera data don't come from the same cameras.
-
- If a single value is provided, it is broadcasted to all cameras.
+ If a single value is provided, it is broadcasted to all cameras.
Args:
camera_to_worlds: Camera to world matrices. Tensor of per-image c2w matrices, in [R | t] format
@@ -79,6 +75,8 @@ class Cameras(TensorDataclass):
distortion_params: OpenCV 6 radial distortion coefficients
camera_type: Type of camera model. This will be an int corresponding to the CameraType enum.
times: Timestamps for each camera
+ metadata: Additional metadata or data needed for interpolation, will mimic shape of the cameras
+ and will be broadcasted to the rays generated from any derivative RaySamples we create with this
"""
camera_to_worlds: TensorType["num_cameras":..., 3, 4]
@@ -91,6 +89,7 @@ class Cameras(TensorDataclass):
distortion_params: Optional[TensorType["num_cameras":..., 6]]
camera_type: TensorType["num_cameras":..., 1]
times: Optional[TensorType["num_cameras", 1]]
+ metadata: Optional[Dict]
def __init__(
self,
@@ -111,6 +110,7 @@ class Cameras(TensorDataclass):
]
] = CameraType.PERSPECTIVE,
times: Optional[TensorType["num_cameras"]] = None,
+ metadata: Optional[Dict] = None,
) -> None:
"""Initializes the Cameras object.
@@ -146,6 +146,8 @@ class Cameras(TensorDataclass):
self.camera_type = self._init_get_camera_type(camera_type)
self.times = self._init_get_times(times)
+ self.metadata = metadata
+
self.__post_init__() # This will do the dataclass post_init and broadcast all the tensors
self._use_nerfacc = strtobool(os.environ.get("INTERSECT_WITH_NERFACC", "TRUE"))
@@ -716,13 +718,23 @@ class Cameras(TensorDataclass):
times = self.times[camera_indices, 0] if self.times is not None else None
+ metadata = (
+ self._apply_fn_to_dict(self.metadata, lambda x: x[true_indices]) if self.metadata is not None else None
+ )
+ if metadata is not None:
+ metadata["directions_norm"] = directions_norm[0].detach()
+ else:
+ metadata = {"directions_norm": directions_norm[0].detach()}
+
+ times = self.times[camera_indices, 0] if self.times is not None else None
+
return RayBundle(
origins=origins,
directions=directions,
pixel_area=pixel_area,
camera_indices=camera_indices,
times=times,
- metadata={"directions_norm": directions_norm[0].detach()},
+ metadata=metadata,
)
def to_json(
diff --git a/nerfstudio/cameras/rays.py b/nerfstudio/cameras/rays.py
index 0184b4c80..61746187c 100644
--- a/nerfstudio/cameras/rays.py
+++ b/nerfstudio/cameras/rays.py
@@ -57,7 +57,7 @@ class Frustums(TensorDataclass):
return pos
def get_start_positions(self) -> TensorType[..., 3]:
- """Calulates "start" position of frustum.
+ """Calculates "start" position of frustum.
Returns:
xyz positions.
@@ -193,7 +193,7 @@ class RayBundle(TensorDataclass):
"""Times at which rays are sampled"""
def set_camera_indices(self, camera_index: int) -> None:
- """Sets all of the the camera indices to a specific camera index.
+ """Sets all the camera indices to a specific camera index.
Args:
camera_index: Camera index.
diff --git a/nerfstudio/configs/base_config.py b/nerfstudio/configs/base_config.py
index 3d7872d74..fd5608b4e 100644
--- a/nerfstudio/configs/base_config.py
+++ b/nerfstudio/configs/base_config.py
@@ -131,19 +131,10 @@ class ViewerConfig(PrintableConfig):
relative_log_filename: str = "viewer_log_filename.txt"
"""Filename to use for the log file."""
- start_train: bool = True
- """whether to immediately start training upon loading viewer
- if False, will just visualize dataset but you can toggle training in viewer"""
- zmq_port: Optional[int] = None
- """The zmq port to connect to for communication. If None, find an available port."""
- launch_bridge_server: bool = True
- """whether or not to launch the bridge server"""
websocket_port: Optional[int] = None
"""The websocket port to connect to. If None, find an available port."""
websocket_port_default: int = 7007
"""The default websocket port to connect to if websocket_port is not specified"""
- ip_address: str = "127.0.0.1"
- """the ip address where the bridge server is running"""
num_rays_per_chunk: int = 32768
"""number of rays per chunk to render with viewer"""
max_num_display_images: int = 512
@@ -155,5 +146,3 @@ class ViewerConfig(PrintableConfig):
"""Image format viewer should use; jpeg is lossy compression, while png is lossless."""
jpeg_quality: int = 90
"""Quality tradeoff to use for jpeg compression."""
- png_compression: int = 1
- """Size/speed tradeoff to use for png compression."""
diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py
index 0b3337d82..5aa3563dc 100644
--- a/nerfstudio/configs/method_configs.py
+++ b/nerfstudio/configs/method_configs.py
@@ -49,9 +49,11 @@ from nerfstudio.engine.optimizers import AdamOptimizerConfig, RAdamOptimizerConf
from nerfstudio.engine.schedulers import (
CosineDecaySchedulerConfig,
ExponentialDecaySchedulerConfig,
+ MultiStepSchedulerConfig,
)
from nerfstudio.engine.trainer import TrainerConfig
from nerfstudio.field_components.temporal_distortions import TemporalDistortionKind
+from nerfstudio.fields.sdf_field import SDFFieldConfig
from nerfstudio.models.depth_nerfacto import DepthNerfactoModelConfig
from nerfstudio.models.dreamembedding import DreamEmbeddingModelConfig
from nerfstudio.models.dreamfusion import DreamFusionModelConfig
@@ -62,6 +64,7 @@ from nerfstudio.models.nerfacto import NerfactoModelConfig
from nerfstudio.models.nerfplayer_nerfacto import NerfplayerNerfactoModelConfig
from nerfstudio.models.nerfplayer_ngp import NerfplayerNGPModelConfig
from nerfstudio.models.neus import NeuSModelConfig
+from nerfstudio.models.neus_facto import NeuSFactoModelConfig
from nerfstudio.models.semantic_nerfw import SemanticNerfWModelConfig
from nerfstudio.models.tensorf import TensoRFModelConfig
from nerfstudio.models.vanilla_nerf import NeRFModel, VanillaModelConfig
@@ -121,6 +124,46 @@ method_configs["nerfacto"] = TrainerConfig(
viewer=ViewerConfig(num_rays_per_chunk=1 << 15),
vis="viewer",
)
+method_configs["nerfacto-big"] = TrainerConfig(
+ method_name="nerfacto",
+ steps_per_eval_batch=500,
+ steps_per_save=2000,
+ max_num_iterations=100000,
+ mixed_precision=True,
+ pipeline=VanillaPipelineConfig(
+ datamanager=VanillaDataManagerConfig(
+ dataparser=NerfstudioDataParserConfig(),
+ train_num_rays_per_batch=4096,
+ eval_num_rays_per_batch=4096,
+ camera_optimizer=CameraOptimizerConfig(
+ mode="SO3xR3", optimizer=RAdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-3)
+ ),
+ ),
+ model=NerfactoModelConfig(
+ eval_num_rays_per_chunk=1 << 15,
+ num_nerf_samples_per_ray=128,
+ num_proposal_samples_per_ray=(512, 256),
+ hidden_dim=128,
+ hidden_dim_color=128,
+ hidden_dim_transient=128,
+ max_res=3000,
+ proposal_weights_anneal_max_num_iters=5000,
+ log2_hashmap_size=21,
+ ),
+ ),
+ optimizers={
+ "proposal_networks": {
+ "optimizer": RAdamOptimizerConfig(lr=1e-2, eps=1e-15),
+ "scheduler": None,
+ },
+ "fields": {
+ "optimizer": RAdamOptimizerConfig(lr=1e-2, eps=1e-15),
+ "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=100000),
+ },
+ },
+ viewer=ViewerConfig(num_rays_per_chunk=1 << 15),
+ vis="viewer",
+)
method_configs["depth-nerfacto"] = TrainerConfig(
method_name="depth-nerfacto",
@@ -602,6 +645,56 @@ method_configs["neus"] = TrainerConfig(
vis="viewer",
)
+method_configs["neus-facto"] = TrainerConfig(
+ method_name="neus-facto",
+ steps_per_eval_image=5000,
+ steps_per_eval_batch=5000,
+ steps_per_save=2000,
+ steps_per_eval_all_images=1000000, # set to a very large model so we don't eval with all images
+ max_num_iterations=20001,
+ mixed_precision=False,
+ pipeline=VanillaPipelineConfig(
+ datamanager=SDFDataManagerConfig(
+ dataparser=SDFStudioDataParserConfig(),
+ train_num_rays_per_batch=2048,
+ eval_num_rays_per_batch=2048,
+ camera_optimizer=CameraOptimizerConfig(
+ mode="SO3xR3", optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2)
+ ),
+ ),
+ model=NeuSFactoModelConfig(
+ # proposal network allows for signifanctly smaller sdf/color network
+ sdf_field=SDFFieldConfig(
+ use_grid_feature=True,
+ num_layers=2,
+ num_layers_color=2,
+ hidden_dim=256,
+ bias=0.5,
+ beta_init=0.8,
+ use_appearance_embedding=False,
+ ),
+ background_model="none",
+ eval_num_rays_per_chunk=2048,
+ ),
+ ),
+ optimizers={
+ "proposal_networks": {
+ "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15),
+ "scheduler": MultiStepSchedulerConfig(max_steps=20001, milestones=(10000, 1500, 18000)),
+ },
+ "fields": {
+ "optimizer": AdamOptimizerConfig(lr=5e-4, eps=1e-15),
+ "scheduler": CosineDecaySchedulerConfig(warm_up_end=500, learning_rate_alpha=0.05, max_steps=20001),
+ },
+ "field_background": {
+ "optimizer": AdamOptimizerConfig(lr=5e-4, eps=1e-15),
+ "scheduler": CosineDecaySchedulerConfig(warm_up_end=500, learning_rate_alpha=0.05, max_steps=20001),
+ },
+ },
+ viewer=ViewerConfig(num_rays_per_chunk=1 << 15),
+ vis="viewer",
+)
+
external_methods, external_descriptions = discover_methods()
method_configs.update(external_methods)
descriptions.update(external_descriptions)
diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py
index 8bcd21bbe..7e2ebedeb 100644
--- a/nerfstudio/data/datamanagers/base_datamanager.py
+++ b/nerfstudio/data/datamanagers/base_datamanager.py
@@ -149,10 +149,11 @@ class DataManager(nn.Module):
Usage:
To get data, use the next_train and next_eval functions.
This data manager's next_train and next_eval methods will return 2 things:
- 1. A Raybundle: This will contain the rays we are sampling, with latents and
- conditionals attached (everything needed at inference)
- 2. A "batch" of auxiliary information: This will contain the mask, the ground truth
- pixels, etc needed to actually train, score, etc the model
+
+ 1. A Raybundle: This will contain the rays we are sampling, with latents and
+ conditionals attached (everything needed at inference)
+ 2. A "batch" of auxiliary information: This will contain the mask, the ground truth
+ pixels, etc needed to actually train, score, etc the model
Rationale:
Because of this abstraction we've added, we can support more NeRF paradigms beyond the
diff --git a/nerfstudio/data/dataparsers/nerfosr_dataparser.py b/nerfstudio/data/dataparsers/nerfosr_dataparser.py
index 820a28956..f7b84b261 100644
--- a/nerfstudio/data/dataparsers/nerfosr_dataparser.py
+++ b/nerfstudio/data/dataparsers/nerfosr_dataparser.py
@@ -42,7 +42,7 @@ from nerfstudio.data.scene_box import SceneBox
CONSOLE = Console(width=120)
-def find_files(directory: str, exts: List[str]):
+def _find_files(directory: str, exts: List[str]):
"""Find all files in a directory that have a certain file extension.
Args:
@@ -94,8 +94,8 @@ def get_camera_params(
split_dir = f"{scene_dir}/{split}"
# camera parameters files
- intrinsics_files = find_files(f"{split_dir}/intrinsics", exts=["*.txt"])
- pose_files = find_files(f"{split_dir}/pose", exts=["*.txt"])
+ intrinsics_files = _find_files(f"{split_dir}/intrinsics", exts=["*.txt"])
+ pose_files = _find_files(f"{split_dir}/pose", exts=["*.txt"])
num_cams = len(pose_files)
@@ -219,12 +219,12 @@ class NeRFOSR(DataParser):
)
# --- images ---
- image_filenames = find_files(f"{split_dir}/rgb", exts=["*.png", "*.jpg", "*.JPG", "*.PNG"])
+ image_filenames = _find_files(f"{split_dir}/rgb", exts=["*.png", "*.jpg", "*.JPG", "*.PNG"])
# --- masks ---
mask_filenames = []
if self.config.use_masks:
- mask_filenames = find_files(f"{split_dir}/mask", exts=["*.png", "*.jpg", "*.JPG", "*.PNG"])
+ mask_filenames = _find_files(f"{split_dir}/mask", exts=["*.png", "*.jpg", "*.JPG", "*.PNG"])
dataparser_outputs = DataparserOutputs(
image_filenames=image_filenames,
diff --git a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py
index 4a74cd961..7b283d747 100644
--- a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py
+++ b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py
@@ -46,7 +46,7 @@ class NerfstudioDataParserConfig(DataParserConfig):
_target: Type = field(default_factory=lambda: Nerfstudio)
"""target class to instantiate"""
- data: Path = Path("data/nerfstudio")
+ data: Path = Path()
"""Directory or explicit json file path specifying location of data."""
scale_factor: float = 1.0
"""How much to scale the camera origins by."""
@@ -76,6 +76,8 @@ class Nerfstudio(DataParser):
def _generate_dataparser_outputs(self, split="train"):
# pylint: disable=too-many-statements
+ assert self.config.data.exists(), f"Data directory {self.config.data} does not exist."
+
if self.config.data.suffix == ".json":
meta = load_from_json(self.config.data)
data_dir = self.config.data.parent
diff --git a/nerfstudio/data/dataparsers/scannet_dataparser.py b/nerfstudio/data/dataparsers/scannet_dataparser.py
index d5a787799..5d2702f18 100644
--- a/nerfstudio/data/dataparsers/scannet_dataparser.py
+++ b/nerfstudio/data/dataparsers/scannet_dataparser.py
@@ -39,10 +39,14 @@ class ScanNetDataParserConfig(DataParserConfig):
ScanNet dataset (https://www.scan-net.org/) is a large-scale 3D dataset of indoor scenes.
This dataparser assumes that the dense stream was extracted from .sens files.
Expected structure of scene directory:
- - color/
- - depth/
- - intrinsic/
- - pose/
+
+ .. code-block:: text
+
+ root/
+ ├── color/
+ ├── depth/
+ ├── intrinsic/
+ ├── pose/
"""
_target: Type = field(default_factory=lambda: ScanNet)
diff --git a/nerfstudio/data/dataparsers/sdfstudio_dataparser.py b/nerfstudio/data/dataparsers/sdfstudio_dataparser.py
index 9b692c65b..97c7a299d 100644
--- a/nerfstudio/data/dataparsers/sdfstudio_dataparser.py
+++ b/nerfstudio/data/dataparsers/sdfstudio_dataparser.py
@@ -51,11 +51,11 @@ class SDFStudioDataParserConfig(DataParserConfig):
scene_scale: float = 2.0
"""
Sets the bounding cube to have edge length of this size.
- The longest dimension of the Friends axis-aligned bbox will be scaled to this value.
+ The longest dimension of the axis-aligned bbox will be scaled to this value.
"""
skip_every_for_val_split: int = 1
"""sub sampling validation images"""
- auto_orient: bool = False
+ auto_orient: bool = True
@dataclass
@@ -141,8 +141,8 @@ class SDFStudio(DataParser):
# TODO supports downsample
# cameras.rescale_output_resolution(scaling_factor=1.0 / self.config.downscale_factor)
-
- assert meta["has_mono_prior"] == self.config.include_mono_prior, f"no mono prior in {self.config.data}"
+ if self.config.include_mono_prior:
+ assert meta["has_mono_prior"], f"no mono prior in {self.config.data}"
dataparser_outputs = DataparserOutputs(
image_filenames=image_filenames,
diff --git a/nerfstudio/data/scene_box.py b/nerfstudio/data/scene_box.py
index e28761e29..e513a1428 100644
--- a/nerfstudio/data/scene_box.py
+++ b/nerfstudio/data/scene_box.py
@@ -17,7 +17,7 @@ Dataset input structures.
"""
from dataclasses import dataclass
-from typing import Dict, Union
+from typing import Union
import torch
from torchtyping import TensorType
@@ -64,21 +64,6 @@ class SceneBox:
normalized_positions = (positions - aabb[0]) / aabb_lengths
return normalized_positions
- def to_json(self) -> Dict:
- """Returns a json object from the Python object."""
- return {"type": "aabb", "min_point": self.aabb[0].tolist(), "max_point": self.aabb[1].tolist()}
-
- @staticmethod
- def from_json(json_: Dict) -> "SceneBox":
- """Returns the an instance of SceneBox from a json dictionary.
-
- Args:
- json_: the json dictionary containing scene box information
- """
- assert json_["type"] == "aabb"
- aabb = torch.tensor([json_[0], json_[1]])
- return SceneBox(aabb=aabb)
-
@staticmethod
def from_camera_poses(poses: TensorType[..., 3, 4], scale_factor: float) -> "SceneBox":
"""Returns the instance of SceneBox that fully envelopes a set of poses
diff --git a/nerfstudio/data/utils/nerfstudio_collate.py b/nerfstudio/data/utils/nerfstudio_collate.py
index ffcd989b4..91774343a 100644
--- a/nerfstudio/data/utils/nerfstudio_collate.py
+++ b/nerfstudio/data/utils/nerfstudio_collate.py
@@ -23,7 +23,6 @@ from typing import Callable, Dict, Union
import torch
import torch.utils.data
-from torch._six import string_classes
from nerfstudio.cameras.cameras import Cameras
@@ -119,7 +118,7 @@ def nerfstudio_collate(
return torch.tensor(batch, dtype=torch.float64)
elif isinstance(elem, int):
return torch.tensor(batch)
- elif isinstance(elem, string_classes):
+ elif isinstance(elem, (str, bytes)):
return batch
elif isinstance(elem, collections.abc.Mapping):
try:
diff --git a/nerfstudio/engine/schedulers.py b/nerfstudio/engine/schedulers.py
index 48a656650..301d9eb5f 100644
--- a/nerfstudio/engine/schedulers.py
+++ b/nerfstudio/engine/schedulers.py
@@ -18,7 +18,7 @@
from abc import abstractmethod
from dataclasses import dataclass, field
-from typing import Optional, Type
+from typing import Optional, Tuple, Type
import numpy as np
from torch.optim import Optimizer, lr_scheduler
@@ -56,6 +56,34 @@ class Scheduler:
"""
+@dataclass
+class MultiStepSchedulerConfig(SchedulerConfig):
+ """Config for multi step scheduler where lr decays by gamma every milestone"""
+
+ _target: Type = field(default_factory=lambda: MultiStepScheduler)
+ """target class to instantiate"""
+ max_steps: int = 1000000
+ """The maximum number of steps."""
+ gamma: float = 0.33
+ """The learning rate decay factor."""
+ milestones: Tuple[int, ...] = (500000, 750000, 900000)
+ """The milestone steps at which to decay the learning rate."""
+
+
+class MultiStepScheduler(Scheduler):
+ """Multi step scheduler where lr decays by gamma every milestone"""
+
+ config: MultiStepSchedulerConfig
+
+ def get_scheduler(self, optimizer: Optimizer, lr_init: float) -> lr_scheduler._LRScheduler:
+ scheduler = lr_scheduler.MultiStepLR(
+ optimizer=optimizer,
+ milestones=self.config.milestones,
+ gamma=self.config.gamma,
+ )
+ return scheduler
+
+
@dataclass
class ExponentialDecaySchedulerConfig(SchedulerConfig):
"""Config for exponential decay scheduler with warmup"""
@@ -111,7 +139,7 @@ class ExponentialDecayScheduler(Scheduler):
@dataclass
class CosineDecaySchedulerConfig(SchedulerConfig):
- """Basic scheduler config with self-defined exponential decay schedule"""
+ """Config for cosine decay schedule"""
_target: Type = field(default_factory=lambda: CosineDecayScheduler)
"""target class to instantiate"""
@@ -124,7 +152,7 @@ class CosineDecaySchedulerConfig(SchedulerConfig):
class CosineDecayScheduler(Scheduler):
- """Starts with a flat lr schedule until it reaches N epochs then applies a given scheduler"""
+ """Cosine decay scheduler with linear warmup"""
config: CosineDecaySchedulerConfig
diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py
index dd34aa7b4..8425cc1a3 100644
--- a/nerfstudio/engine/trainer.py
+++ b/nerfstudio/engine/trainer.py
@@ -23,6 +23,7 @@ import os
import time
from dataclasses import dataclass, field
from pathlib import Path
+from threading import Lock
from typing import Dict, List, Optional, Tuple, Type, Union
import torch
@@ -46,7 +47,7 @@ from nerfstudio.utils.decorators import (
)
from nerfstudio.utils.misc import step_check
from nerfstudio.utils.writer import EventName, TimeWriter
-from nerfstudio.viewer.server import viewer_utils
+from nerfstudio.viewer.server.viewer_state import ViewerState
CONSOLE = Console(width=120)
@@ -103,6 +104,7 @@ class Trainer:
pipeline: The pipeline object.
optimizers: The optimizers object.
callbacks: The callbacks object.
+ is_training: Whether the model is training.
"""
pipeline: VanillaPipeline
@@ -110,11 +112,13 @@ class Trainer:
callbacks: List[TrainingCallback]
def __init__(self, config: TrainerConfig, local_rank: int = 0, world_size: int = 1) -> None:
+ self.train_lock = Lock()
self.config = config
self.local_rank = local_rank
self.world_size = world_size
self.device: TORCH_DEVICE = "cpu" if world_size == 0 else f"cuda:{local_rank}"
self.mixed_precision: bool = self.config.mixed_precision
+ self.is_training: bool = True
if self.device == "cpu":
self.mixed_precision = False
CONSOLE.print("Mixed precision is disabled for CPU training.")
@@ -138,6 +142,11 @@ class Trainer:
'test': loads train/test datasets into memory
'inference': does not load any dataset into memory
"""
+ self.pipeline = self.config.pipeline.setup(
+ device=self.device, test_mode=test_mode, world_size=self.world_size, local_rank=self.local_rank
+ )
+ self.optimizers = self.setup_optimizers()
+
# set up viewer if enabled
viewer_log_path = self.base_dir / self.config.viewer.relative_log_filename
self.viewer_state, banner_messages = None, None
@@ -145,9 +154,15 @@ class Trainer:
datapath = self.config.data
if datapath is None:
datapath = self.base_dir
- self.viewer_state, banner_messages = viewer_utils.setup_viewer(
- self.config.viewer, log_filename=viewer_log_path, datapath=datapath
+ self.viewer_state = ViewerState(
+ self.config.viewer,
+ log_filename=viewer_log_path,
+ datapath=datapath,
+ pipeline=self.pipeline,
+ trainer=self,
+ train_lock=self.train_lock,
)
+ banner_messages = [f"Viewer at: {self.viewer_state.viewer_url}"]
self._check_viewer_warnings()
self.pipeline = self.config.pipeline.setup(
@@ -211,21 +226,26 @@ class Trainer:
num_iterations = self.config.max_num_iterations
step = 0
for step in range(self._start_step, self._start_step + num_iterations):
- with TimeWriter(writer, EventName.ITER_TRAIN_TIME, step=step) as train_t:
- self.pipeline.train()
+ while not self.is_training:
+ time.sleep(0.01)
+ with self.train_lock:
+ with TimeWriter(writer, EventName.ITER_TRAIN_TIME, step=step) as train_t:
+ self.pipeline.train()
- # training callbacks before the training iteration
- for callback in self.callbacks:
- callback.run_callback_at_location(
- step, location=TrainingCallbackLocation.BEFORE_TRAIN_ITERATION
- )
+ # training callbacks before the training iteration
+ for callback in self.callbacks:
+ callback.run_callback_at_location(
+ step, location=TrainingCallbackLocation.BEFORE_TRAIN_ITERATION
+ )
- # time the forward pass
- loss, loss_dict, metrics_dict = self.train_iteration(step)
+ # time the forward pass
+ loss, loss_dict, metrics_dict = self.train_iteration(step)
- # training callbacks after the training iteration
- for callback in self.callbacks:
- callback.run_callback_at_location(step, location=TrainingCallbackLocation.AFTER_TRAIN_ITERATION)
+ # training callbacks after the training iteration
+ for callback in self.callbacks:
+ callback.run_callback_at_location(
+ step, location=TrainingCallbackLocation.AFTER_TRAIN_ITERATION
+ )
# Skip the first two steps to avoid skewed timings that break the viewer rendering speed estimate.
if step > 1:
@@ -243,6 +263,14 @@ class Trainer:
writer.put_scalar(name="Train Loss", scalar=loss, step=step)
writer.put_dict(name="Train Loss Dict", scalar_dict=loss_dict, step=step)
writer.put_dict(name="Train Metrics Dict", scalar_dict=metrics_dict, step=step)
+ # The actual memory allocated by Pytorch. This is likely less than the amount
+ # shown in nvidia-smi since some unused memory can be held by the caching
+ # allocator and some context needs to be created on GPU. See Memory management
+ # (https://pytorch.org/docs/stable/notes/cuda.html#cuda-memory-management)
+ # for more details about GPU memory management.
+ writer.put_scalar(
+ name="GPU Memory (MB)", scalar=torch.cuda.max_memory_allocated() / (1024**2), step=step
+ )
# Do not perform evaluation if there are no validation images
if self.pipeline.datamanager.eval_dataset:
@@ -263,14 +291,8 @@ class Trainer:
CONSOLE.print("[bold green]:tada: :tada: :tada: Training Finished :tada: :tada: :tada:", justify="center")
if not self.config.viewer.quit_on_train_completion:
CONSOLE.print("Use ctrl+c to quit", justify="center")
- self._always_render(step)
-
- @check_main_thread
- def _always_render(self, step: int) -> None:
- if self.viewer_state is not None:
while True:
- self.viewer_state.vis["renderingState/isTraining"].write(False)
- self._update_viewer_state(step)
+ time.sleep(0.01)
@check_main_thread
def _check_viewer_warnings(self) -> None:
@@ -292,10 +314,8 @@ class Trainer:
assert self.viewer_state and self.pipeline.datamanager.train_dataset
self.viewer_state.init_scene(
dataset=self.pipeline.datamanager.train_dataset,
- start_train=self.config.viewer.start_train,
+ start_train=True,
)
- if not self.config.viewer.start_train:
- self._always_render(self._start_step)
@check_viewer_enabled
def _update_viewer_state(self, step: int) -> None:
@@ -306,16 +326,12 @@ class Trainer:
step: current train step
"""
assert self.viewer_state is not None
- with TimeWriter(writer, EventName.ITER_VIS_TIME, step=step) as _:
- num_rays_per_batch: int = self.pipeline.datamanager.get_train_rays_per_batch()
- try:
- self.viewer_state.update_scene(self, step, self.pipeline.model, num_rays_per_batch)
- except RuntimeError:
- time.sleep(0.03) # sleep to allow buffer to reset
- assert self.viewer_state.vis is not None
- self.viewer_state.vis["renderingState/log_errors"].write(
- "Error: GPU out of memory. Reduce resolution to prevent viewer from crashing."
- )
+ num_rays_per_batch: int = self.pipeline.datamanager.get_train_rays_per_batch()
+ try:
+ self.viewer_state.update_scene(step, num_rays_per_batch)
+ except RuntimeError:
+ time.sleep(0.03) # sleep to allow buffer to reset
+ CONSOLE.log("Viewer failed. Continuing training.")
@check_viewer_enabled
def _update_viewer_rays_per_sec(self, train_t: TimeWriter, vis_t: TimeWriter, step: int) -> None:
@@ -392,8 +408,10 @@ class Trainer:
Args:
step: Current training step.
"""
+
self.optimizers.zero_grad_all()
cpu_or_cuda_str: str = self.device.split(":")[0]
+
with torch.autocast(device_type=cpu_or_cuda_str, enabled=self.mixed_precision):
_, loss_dict, metrics_dict = self.pipeline.get_train_loss_dict(step=step)
loss = functools.reduce(torch.add, loss_dict.values())
diff --git a/nerfstudio/exporter/marching_cubes.py b/nerfstudio/exporter/marching_cubes.py
new file mode 100644
index 000000000..03cd7da86
--- /dev/null
+++ b/nerfstudio/exporter/marching_cubes.py
@@ -0,0 +1,253 @@
+# Copyright 2022 The Nerfstudio Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module implements the Marching Cubes algorithm for extracting
+isosurfaces
+"""
+
+from typing import Callable, List, Tuple, Union
+
+import numpy as np
+import torch
+import trimesh
+from skimage import measure
+from torchtyping import TensorType
+
+avg_pool_3d = torch.nn.AvgPool3d(2, stride=2)
+upsample = torch.nn.Upsample(scale_factor=2, mode="nearest")
+
+
+def create_point_pyramid(
+ points: TensorType[3, "height", "width", "depth"]
+) -> List[TensorType[3, "height", "width", "depth"]]:
+ """
+ Create a point pyramid for multi-resolution evaluation.
+
+ Args:
+ points: A torch tensor containing 3D points.
+
+ Returns:
+ A list of torch tensors representing points at different resolutions.
+ """
+ points_pyramid = [points]
+ for _ in range(3):
+ points = avg_pool_3d(points[None])[0]
+ points_pyramid.append(points)
+ points_pyramid = points_pyramid[::-1]
+ return points_pyramid
+
+
+def evaluate_sdf(sdf: Callable, points: TensorType["batch", 3]) -> TensorType["batch"]:
+ """
+ Evaluate a signed distance function (SDF) for a batch of points.
+
+ Args:
+ sdf: A callable function that takes a tensor of size (N, 3) containing
+ 3D points and returns a tensor of size (N,) with the SDF values.
+ points: A torch tensor containing 3D points.
+
+ Returns:
+ A torch tensor with the SDF values evaluated at the given points.
+ """
+ z = []
+ for _, pnts in enumerate(torch.split(points, 100000, dim=0)):
+ z.append(sdf(pnts))
+ z = torch.cat(z, axis=0)
+ return z
+
+
+def evaluate_multiresolution_sdf(
+ evaluate: Callable,
+ points_pyramid: List[TensorType[3, "height", "width", "depth"]],
+ coarse_mask: Union[TensorType[1, 1, "height", "width", "depth"], None],
+ x_max: float,
+ x_min: float,
+ crop_n: int,
+) -> TensorType["batch"]:
+
+ """
+ Evaluate SDF values using a multi-resolution approach with a given point pyramid.
+
+ Args:
+ evaluate: A callable function to evaluate SDF values at given points.
+ points_pyramid: A list of torch tensors representing points at different resolutions.
+ coarse_mask: A binary mask tensor indicating regions of the bounding box where the SDF
+ is expected to have a zero-crossing.
+ x_max: The maximum x-coordinate of the bounding box.
+ x_min: The minimum x-coordinate of the bounding box.
+ crop_n: The resolution of the grid used to sample the SDF.
+
+ Returns:
+ A torch tensor with the SDF values evaluated at the given points.
+ """
+ mask = None
+ threshold = 2 * (x_max - x_min) / crop_n * 8
+ for pid, pts in enumerate(points_pyramid):
+ coarse_n = pts.shape[-1]
+ pts = pts.reshape(3, -1).permute(1, 0).contiguous()
+
+ if mask is None:
+ # Only evaluate SDF
+ if coarse_mask is not None:
+ pts_sdf = torch.ones_like(pts[:, 1])
+ valid_mask = torch.nn.functional.grid_sample(coarse_mask, pts[None, None, None])[0, 0, 0, 0] > 0
+ if valid_mask.any():
+ pts_sdf[valid_mask] = evaluate(pts[valid_mask].contiguous())
+ else:
+ pts_sdf = evaluate(pts)
+ else:
+ mask = mask.reshape(-1)
+ pts_to_eval = pts[mask]
+
+ if pts_to_eval.shape[0] > 0:
+ pts_sdf_eval = evaluate(pts_to_eval.contiguous())
+ pts_sdf[mask] = pts_sdf_eval
+
+ if pid < 3:
+ # Update mask
+ mask = torch.abs(pts_sdf) < threshold
+ mask = mask.reshape(coarse_n, coarse_n, coarse_n)[None, None]
+ mask = upsample(mask.float()).bool()
+
+ pts_sdf = pts_sdf.reshape(coarse_n, coarse_n, coarse_n)[None, None]
+ pts_sdf = upsample(pts_sdf)
+ pts_sdf = pts_sdf.reshape(-1)
+
+ threshold /= 2.0
+ return pts_sdf
+
+
+@torch.no_grad()
+def generate_mesh_with_multires_marching_cubes(
+ geometry_callable_field: Callable,
+ resolution: int = 512,
+ bounding_box_min: Tuple[float, float, float] = (-1.0, -1.0, -1.0),
+ bounding_box_max: Tuple[float, float, float] = (1.0, 1.0, 1.0),
+ isosurface_threshold: float = 0.0,
+ coarse_mask: Union[None, TensorType["height", "width", "depth"]] = None,
+) -> trimesh.Trimesh:
+ """
+ Computes the isosurface of a signed distance function (SDF) defined by the
+ callable `sdf` in a given bounding box with a specified resolution. The SDF
+ is sampled at a set of points within a regular grid, and the marching cubes
+ algorithm is used to generate a mesh that approximates the isosurface at a
+ specified isovalue `level`.
+
+ Args:
+ sdf: A callable function that takes as input a tensor of size
+ (N, 3) containing 3D points, and returns a tensor of size (N,) containing
+ the signed distance function evaluated at those points.
+ output_path: The output directory where the resulting mesh will be saved.
+ resolution: The resolution of the grid used to sample the SDF.
+ bounding_box_min: The minimum coordinates of the bounding box in which the SDF
+ will be evaluated.
+ bounding_box_max: The maximum coordinates of the bounding box in which the SDF
+ will be evaluated.
+ isosurface_threshold: The isovalue at which to approximate the isosurface.
+ coarse_mask: A binary mask tensor of size ("height", "width", "depth") that indicates the regions
+ of the bounding box where the SDF is expected to have a zero-crossing. If
+ provided, the algorithm first evaluates the SDF at the coarse voxels where
+ the mask is True, and then refines the evaluation within these voxels using
+ a multi-scale approach. If None, evaluates the SDF at all points in the
+ bounding box.
+ Returns:
+ A torch tensor with the SDF values evaluated at the given points.
+ """
+ # Check if resolution is divisible by 512
+ assert (
+ resolution % 512 == 0
+ ), f"""resolution must be divisible by 512, got {resolution}.
+ This is important because the algorithm uses a multi-resolution approach
+ to evaluate the SDF where the mimimum resolution is 512."""
+ # Prepare coarse mask if provided
+ if coarse_mask is not None:
+ coarse_mask = coarse_mask.permute(2, 1, 0)[None, None].cuda().float()
+
+ # Initialize variables
+ crop_n = 512
+ N = resolution // crop_n
+ grid_min = bounding_box_min
+ grid_max = bounding_box_max
+ xs = np.linspace(grid_min[0], grid_max[0], N + 1)
+ ys = np.linspace(grid_min[1], grid_max[1], N + 1)
+ zs = np.linspace(grid_min[2], grid_max[2], N + 1)
+
+ # Initialize meshes list
+ meshes = []
+
+ # Iterate over the grid
+ for i in range(N):
+ for j in range(N):
+ for k in range(N):
+ # Calculate grid cell boundaries
+ x_min, x_max = xs[i], xs[i + 1]
+ y_min, y_max = ys[j], ys[j + 1]
+ z_min, z_max = zs[k], zs[k + 1]
+
+ # Create point grid
+ x = np.linspace(x_min, x_max, crop_n)
+ y = np.linspace(y_min, y_max, crop_n)
+ z = np.linspace(z_min, z_max, crop_n)
+ xx, yy, zz = np.meshgrid(x, y, z, indexing="ij")
+ points = torch.tensor(np.vstack([xx.ravel(), yy.ravel(), zz.ravel()]).T, dtype=torch.float).cuda()
+
+ # Function to evaluate SDF for a batch of points
+ def evaluate(points: torch.Tensor) -> torch.Tensor:
+ return evaluate_sdf(geometry_callable_field, points)
+
+ # Construct point pyramids
+ points = points.reshape(crop_n, crop_n, crop_n, 3).permute(3, 0, 1, 2)
+ if coarse_mask is not None:
+ points_tmp = points.permute(1, 2, 3, 0)[None].cuda()
+ current_mask = torch.nn.functional.grid_sample(coarse_mask, points_tmp)
+ current_mask = (current_mask > 0.0).cpu().numpy()[0, 0]
+ else:
+ current_mask = None
+
+ # Create point pyramid for multi-resolution evaluation
+ points_pyramid = create_point_pyramid(points)
+
+ # Evaluate SDF using multi-resolution approach and mask
+ pts_sdf = evaluate_multiresolution_sdf(evaluate, points_pyramid, coarse_mask, x_max, x_min, crop_n)
+
+ z = pts_sdf.detach().cpu().numpy()
+
+ # Skip if no surface found
+ if current_mask is not None:
+ valid_z = z.reshape(crop_n, crop_n, crop_n)[current_mask]
+ if valid_z.shape[0] <= 0 or (
+ np.min(valid_z) > isosurface_threshold or np.max(valid_z) < isosurface_threshold
+ ):
+ continue
+
+ if not (np.min(z) > isosurface_threshold or np.max(z) < isosurface_threshold):
+ z = z.astype(np.float32)
+ verts, faces, normals, _ = measure.marching_cubes(
+ volume=z.reshape(crop_n, crop_n, crop_n),
+ level=isosurface_threshold,
+ spacing=(
+ (x_max - x_min) / (crop_n - 1),
+ (y_max - y_min) / (crop_n - 1),
+ (z_max - z_min) / (crop_n - 1),
+ ),
+ mask=current_mask,
+ )
+ verts = verts + np.array([x_min, y_min, z_min])
+
+ meshcrop = trimesh.Trimesh(verts, faces, normals)
+ meshes.append(meshcrop)
+
+ combined_mesh: trimesh.Trimesh = trimesh.util.concatenate(meshes)
+ return combined_mesh
diff --git a/nerfstudio/exporter/texture_utils.py b/nerfstudio/exporter/texture_utils.py
index 4001d1c60..d7e08c669 100644
--- a/nerfstudio/exporter/texture_utils.py
+++ b/nerfstudio/exporter/texture_utils.py
@@ -401,6 +401,7 @@ def export_textured_mesh(
camera_indices = torch.zeros_like(origins[..., 0:1])
nears = torch.zeros_like(origins[..., 0:1])
fars = torch.ones_like(origins[..., 0:1]) * raylen
+ directions_norm = torch.ones_like(origins[..., 0:1]) # for surface model
camera_ray_bundle = RayBundle(
origins=origins,
directions=directions,
@@ -408,6 +409,7 @@ def export_textured_mesh(
camera_indices=camera_indices,
nears=nears,
fars=fars,
+ metadata={"directions_norm": directions_norm},
)
CONSOLE.print("Creating texture image by rendering with NeRF...")
diff --git a/nerfstudio/field_components/encodings.py b/nerfstudio/field_components/encodings.py
index 462080287..ac7cf2aa1 100644
--- a/nerfstudio/field_components/encodings.py
+++ b/nerfstudio/field_components/encodings.py
@@ -141,9 +141,9 @@ class NeRFEncoding(Encoding):
Returns:
Output values will be between -1 and 1
"""
- in_tensor = 2 * torch.pi * in_tensor # scale to [0, 2pi]
+ scaled_in_tensor = 2 * torch.pi * in_tensor # scale to [0, 2pi]
freqs = 2 ** torch.linspace(self.min_freq, self.max_freq, self.num_frequencies).to(in_tensor.device)
- scaled_inputs = in_tensor[..., None] * freqs # [..., "input_dim", "num_scales"]
+ scaled_inputs = scaled_in_tensor[..., None] * freqs # [..., "input_dim", "num_scales"]
scaled_inputs = scaled_inputs.view(*scaled_inputs.shape[:-2], -1) # [..., "input_dim" * "num_scales"]
if covs is None:
@@ -201,8 +201,8 @@ class RFFEncoding(Encoding):
Returns:
Output values will be between -1 and 1
"""
- in_tensor = 2 * torch.pi * in_tensor # scale to [0, 2pi]
- scaled_inputs = in_tensor @ self.b_matrix # [..., "num_frequencies"]
+ scaled_in_tensor = 2 * torch.pi * in_tensor # scale to [0, 2pi]
+ scaled_inputs = scaled_in_tensor @ self.b_matrix # [..., "num_frequencies"]
if covs is None:
encoded_inputs = torch.sin(torch.cat([scaled_inputs, scaled_inputs + torch.pi / 2.0], dim=-1))
diff --git a/nerfstudio/models/neus_facto.py b/nerfstudio/models/neus_facto.py
new file mode 100644
index 000000000..554f0d90f
--- /dev/null
+++ b/nerfstudio/models/neus_facto.py
@@ -0,0 +1,222 @@
+# Copyright 2022 The Nerfstudio Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Implementation of NeuS similar to nerfacto where proposal sampler is used.
+Based on SDFStudio https://github.com/autonomousvision/sdfstudio/
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Tuple, Type
+
+import numpy as np
+import torch
+from torch.nn import Parameter
+
+from nerfstudio.cameras.rays import RayBundle
+from nerfstudio.engine.callbacks import (
+ TrainingCallback,
+ TrainingCallbackAttributes,
+ TrainingCallbackLocation,
+)
+from nerfstudio.field_components.field_heads import FieldHeadNames
+from nerfstudio.fields.density_fields import HashMLPDensityField
+from nerfstudio.model_components.losses import interlevel_loss
+from nerfstudio.model_components.ray_samplers import (
+ ProposalNetworkSampler,
+ UniformSampler,
+)
+from nerfstudio.models.neus import NeuSModel, NeuSModelConfig
+from nerfstudio.utils import colormaps
+
+
+@dataclass
+class NeuSFactoModelConfig(NeuSModelConfig):
+ """NeusFacto Model Config"""
+
+ _target: Type = field(default_factory=lambda: NeuSFactoModel)
+ num_proposal_samples_per_ray: Tuple[int, int] = (256, 96)
+ """Number of samples per ray for the proposal network."""
+ num_neus_samples_per_ray: int = 48
+ """Number of samples per ray for the nerf network."""
+ proposal_update_every: int = 5
+ """Sample every n steps after the warmup"""
+ proposal_warmup: int = 5000
+ """Scales n from 1 to proposal_update_every over this many steps"""
+ num_proposal_iterations: int = 2
+ """Number of proposal network iterations."""
+ use_same_proposal_network: bool = False
+ """Use the same proposal network. Otherwise use different ones."""
+ proposal_net_args_list: List[Dict] = field(
+ default_factory=lambda: [
+ {"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 64},
+ {"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 256},
+ ]
+ )
+ """Arguments for the proposal density fields."""
+ interlevel_loss_mult: float = 1.0
+ """Proposal loss multiplier."""
+ use_proposal_weight_anneal: bool = True
+ """Whether to use proposal weight annealing."""
+ proposal_weights_anneal_slope: float = 10.0
+ """Slope of the annealing function for the proposal weights."""
+ proposal_weights_anneal_max_num_iters: int = 1000
+ """Max num iterations for the annealing function."""
+ use_single_jitter: bool = True
+ """Whether use single jitter or not for the proposal networks."""
+
+
+class NeuSFactoModel(NeuSModel):
+ """NeuSFactoModel extends NeuSModel for a more efficient sampling strategy.
+
+ The model improves the rendering speed and quality by incorporating a learning-based
+ proposal distribution to guide the sampling process.(similar to mipnerf-360)
+
+ Args:
+ config: NeuS configuration to instantiate model
+ """
+
+ config: NeuSFactoModelConfig
+
+ def populate_modules(self):
+ """Instantiate modules and fields, including proposal networks."""
+ super().populate_modules()
+
+ self.density_fns = []
+ num_prop_nets = self.config.num_proposal_iterations
+ # Build the proposal network(s)
+ self.proposal_networks = torch.nn.ModuleList()
+ if self.config.use_same_proposal_network:
+ assert len(self.config.proposal_net_args_list) == 1, "Only one proposal network is allowed."
+ prop_net_args = self.config.proposal_net_args_list[0]
+ network = HashMLPDensityField(
+ self.scene_box.aabb, spatial_distortion=self.scene_contraction, **prop_net_args
+ )
+ self.proposal_networks.append(network)
+ self.density_fns.extend([network.density_fn for _ in range(num_prop_nets)])
+ else:
+ for i in range(num_prop_nets):
+ prop_net_args = self.config.proposal_net_args_list[min(i, len(self.config.proposal_net_args_list) - 1)]
+ network = HashMLPDensityField(
+ self.scene_box.aabb,
+ spatial_distortion=self.scene_contraction,
+ **prop_net_args,
+ )
+ self.proposal_networks.append(network)
+ self.density_fns.extend([network.density_fn for network in self.proposal_networks])
+
+ # update proposal network every iterations
+ update_schedule = lambda step: -1
+
+ initial_sampler = UniformSampler(single_jitter=self.config.use_single_jitter)
+ self.proposal_sampler = ProposalNetworkSampler(
+ num_nerf_samples_per_ray=self.config.num_neus_samples_per_ray,
+ num_proposal_samples_per_ray=self.config.num_proposal_samples_per_ray,
+ num_proposal_network_iterations=self.config.num_proposal_iterations,
+ single_jitter=self.config.use_single_jitter,
+ update_sched=update_schedule,
+ initial_sampler=initial_sampler,
+ )
+
+ def get_param_groups(self) -> Dict[str, List[Parameter]]:
+ """Return a dictionary with the parameters of the proposal networks."""
+ param_groups = super().get_param_groups()
+ param_groups["proposal_networks"] = list(self.proposal_networks.parameters())
+ return param_groups
+
+ def get_training_callbacks(
+ self, training_callback_attributes: TrainingCallbackAttributes
+ ) -> List[TrainingCallback]:
+ callbacks = super().get_training_callbacks(training_callback_attributes)
+
+ if self.config.use_proposal_weight_anneal:
+ # anneal the weights of the proposal network before doing PDF sampling
+ N = self.config.proposal_weights_anneal_max_num_iters
+
+ def set_anneal(step: int):
+ # https://arxiv.org/pdf/2111.12077.pdf eq. 18
+ train_frac = np.clip(step / N, 0, 1)
+ bias = lambda x, b: (b * x) / ((b - 1) * x + 1)
+ anneal = bias(train_frac, self.config.proposal_weights_anneal_slope)
+ self.proposal_sampler.set_anneal(anneal)
+
+ callbacks.append(
+ TrainingCallback(
+ where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION],
+ update_every_num_iters=1,
+ func=set_anneal,
+ )
+ )
+ callbacks.append(
+ TrainingCallback(
+ where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION],
+ update_every_num_iters=1,
+ func=self.proposal_sampler.step_cb,
+ )
+ )
+
+ return callbacks
+
+ def sample_and_forward_field(self, ray_bundle: RayBundle) -> Dict[str, Any]:
+ """Sample rays using proposal networks and compute the corresponding field outputs."""
+ ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns)
+
+ field_outputs = self.field(ray_samples, return_alphas=True)
+ weights, transmittance = ray_samples.get_weights_and_transmittance_from_alphas(
+ field_outputs[FieldHeadNames.ALPHA]
+ )
+ bg_transmittance = transmittance[:, -1, :]
+
+ weights_list.append(weights)
+ ray_samples_list.append(ray_samples)
+
+ samples_and_field_outputs = {
+ "ray_samples": ray_samples,
+ "field_outputs": field_outputs,
+ "weights": weights,
+ "bg_transmittance": bg_transmittance,
+ "weights_list": weights_list,
+ "ray_samples_list": ray_samples_list,
+ }
+ return samples_and_field_outputs
+
+ def get_loss_dict(
+ self, outputs: Dict[str, Any], batch: Dict[str, Any], metrics_dict: Optional[Dict[str, Any]] = None
+ ) -> Dict[str, Any]:
+ """Compute the loss dictionary, including interlevel loss for proposal networks."""
+ loss_dict = super().get_loss_dict(outputs, batch, metrics_dict)
+
+ if self.training:
+ loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss(
+ outputs["weights_list"], outputs["ray_samples_list"]
+ )
+
+ return loss_dict
+
+ def get_image_metrics_and_images(
+ self, outputs: Dict[str, Any], batch: Dict[str, Any]
+ ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]:
+ """Compute image metrics and images, including the proposal depth for each iteration."""
+ metrics_dict, images_dict = super().get_image_metrics_and_images(outputs, batch)
+ for i in range(self.config.num_proposal_iterations):
+ key = f"prop_depth_{i}"
+ prop_depth_i = colormaps.apply_depth_colormap(
+ outputs[key],
+ accumulation=outputs["accumulation"],
+ )
+ images_dict[key] = prop_depth_i
+
+ return metrics_dict, images_dict
diff --git a/nerfstudio/utils/eval_utils.py b/nerfstudio/utils/eval_utils.py
index 3431b20a3..bd535af7f 100644
--- a/nerfstudio/utils/eval_utils.py
+++ b/nerfstudio/utils/eval_utils.py
@@ -33,13 +33,15 @@ from nerfstudio.pipelines.base_pipeline import Pipeline
CONSOLE = Console(width=120)
-def eval_load_checkpoint(config: TrainerConfig, pipeline: Pipeline) -> Path:
+def eval_load_checkpoint(config: TrainerConfig, pipeline: Pipeline) -> Tuple[Path, int]:
## TODO: ideally eventually want to get this to be the same as whatever is used to load train checkpoint too
"""Helper function to load checkpointed pipeline
Args:
config (DictConfig): Configuration of pipeline to load
pipeline (Pipeline): Pipeline instance of which to load weights
+ Returns:
+ A tuple of the path to the loaded checkpoint and the step at which it was saved.
"""
assert config.load_dir is not None
if config.load_step is None:
@@ -61,14 +63,14 @@ def eval_load_checkpoint(config: TrainerConfig, pipeline: Pipeline) -> Path:
loaded_state = torch.load(load_path, map_location="cpu")
pipeline.load_pipeline(loaded_state["pipeline"], loaded_state["step"])
CONSOLE.print(f":white_check_mark: Done loading checkpoint from {load_path}")
- return load_path
+ return load_path, load_step
def eval_setup(
config_path: Path,
eval_num_rays_per_chunk: Optional[int] = None,
test_mode: Literal["test", "val", "inference"] = "test",
-) -> Tuple[TrainerConfig, Pipeline, Path]:
+) -> Tuple[TrainerConfig, Pipeline, Path, int]:
"""Shared setup for loading a saved pipeline for evaluation.
Args:
@@ -81,7 +83,7 @@ def eval_setup(
Returns:
- Loaded config, pipeline module, and corresponding checkpoint.
+ Loaded config, pipeline module, corresponding checkpoint, and step
"""
# load save config
config = yaml.load(config_path.read_text(), Loader=yaml.Loader)
@@ -102,6 +104,6 @@ def eval_setup(
pipeline.eval()
# load checkpointed information
- checkpoint_path = eval_load_checkpoint(config, pipeline)
+ checkpoint_path, step = eval_load_checkpoint(config, pipeline)
- return config, pipeline, checkpoint_path
+ return config, pipeline, checkpoint_path, step
diff --git a/nerfstudio/utils/writer.py b/nerfstudio/utils/writer.py
index bffb0bd03..4d168d508 100644
--- a/nerfstudio/utils/writer.py
+++ b/nerfstudio/utils/writer.py
@@ -18,6 +18,7 @@ Generic Writer class
from __future__ import annotations
import enum
+import os
from abc import abstractmethod
from pathlib import Path
from time import time
@@ -46,7 +47,6 @@ class EventName(enum.Enum):
ITER_TRAIN_TIME = "Train Iter (time)"
TOTAL_TRAIN_TIME = "Train Total (time)"
- ITER_VIS_TIME = "Viewer Rendering (time)"
ETA = "ETA (time)"
TRAIN_RAYS_PER_SEC = "Train Rays / Sec"
TEST_RAYS_PER_SEC = "Test Rays / Sec"
@@ -283,7 +283,12 @@ class WandbWriter(Writer):
"""WandDB Writer Class"""
def __init__(self, log_dir: Path):
- wandb.init(project="nerfstudio-project", dir=str(log_dir), reinit=True, name=log_dir.name)
+ wandb.init(
+ project=os.environ.get("WANDB_PROJECT", "nerfstudio-project"),
+ dir=os.environ.get("WANDB_DIR", str(log_dir)),
+ name=os.environ.get("WANDB_NAME", log_dir.name),
+ reinit=True,
+ )
def write_image(self, name: str, image: TensorType["H", "W", "C"], step: int) -> None:
image = torch.permute(image, (2, 0, 1))
diff --git a/nerfstudio/viewer/app/.eslintrc.json b/nerfstudio/viewer/app/.eslintrc.json
index ecb608b57..f219c3898 100644
--- a/nerfstudio/viewer/app/.eslintrc.json
+++ b/nerfstudio/viewer/app/.eslintrc.json
@@ -4,10 +4,12 @@
"es2021": true
},
"extends": [
+ "react-app",
"eslint:recommended",
"plugin:react/recommended",
"airbnb",
- "prettier"
+ "prettier",
+ "plugin:import/typescript"
],
"parser": "@typescript-eslint/parser",
"parserOptions": {
@@ -27,6 +29,20 @@
"prefer-destructuring": "off",
"react/destructuring-assignment": "off",
"react/prop-types": 0,
- "unused-imports/no-unused-imports-ts": 2
+ "react/jsx-filename-extension": [
+ 2,
+ { "extensions": [".js", ".jsx", ".ts", ".tsx"] }
+ ],
+ "unused-imports/no-unused-imports-ts": 2,
+ "import/extensions": [
+ "error",
+ "ignorePackages",
+ {
+ "js": "never",
+ "jsx": "never",
+ "ts": "never",
+ "tsx": "never"
+ }
+ ]
}
}
diff --git a/nerfstudio/viewer/app/package.json b/nerfstudio/viewer/app/package.json
index 3525ebd86..349b25249 100644
--- a/nerfstudio/viewer/app/package.json
+++ b/nerfstudio/viewer/app/package.json
@@ -1,7 +1,7 @@
{
"name": "viewer",
"homepage": ".",
- "version": "23-03-9-0",
+ "version": "23-04-10-0",
"private": true,
"dependencies": {
"@emotion/react": "^11.10.4",
@@ -15,6 +15,7 @@
"@testing-library/jest-dom": "^5.16.4",
"@testing-library/react": "^13.3.0",
"@testing-library/user-event": "^14.2.0",
+ "await-lock": "^2.2.2",
"camera-controls": "^1.37.2",
"classnames": "^2.3.1",
"dat.gui": "^0.7.9",
@@ -22,8 +23,10 @@
"eslint-config-prettier": "^8.5.0",
"eslint-plugin-unused-imports": "^2.0.0",
"leva": "^0.9.29",
+ "lodash": "^4.17.21",
"meshline": "^2.0.4",
"msgpack-lite": "^0.1.26",
+ "msgpackr": "^1.8.5",
"prop-types": "^15.8.1",
"re-resizable": "^6.9.9",
"react": "^18.1.0",
@@ -77,7 +80,7 @@
"eslint-plugin-react-hooks": "^4.3.0",
"prettier": "2.7.1",
"react-scripts": "^5.0.1",
- "typescript": "^4.7.3",
+ "typescript": "^5.0.3",
"wait-on": "^6.0.1"
},
"resolutions": {
diff --git a/nerfstudio/viewer/app/src/App.jsx b/nerfstudio/viewer/app/src/App.jsx
index 1d2987107..fd42a37f9 100644
--- a/nerfstudio/viewer/app/src/App.jsx
+++ b/nerfstudio/viewer/app/src/App.jsx
@@ -1,14 +1,11 @@
import { CssBaseline, ThemeProvider } from '@mui/material';
import React from 'react';
-import {
- SceneTreeWebSocketListener,
- get_scene_tree,
-} from './modules/Scene/Scene';
+import { get_scene_tree } from './modules/Scene/Scene';
import Banner from './modules/Banner';
import { BasicTabs } from './modules/SidePanel/SidePanel';
import ViewerWindow from './modules/ViewerWindow/ViewerWindow';
-import { appTheme } from './themes/theme.ts';
+import { appTheme } from './themes/theme';
export default function App() {
// The scene tree won't rerender but it will listen to changes
@@ -20,8 +17,6 @@ export default function App() {
- {/* Listens for websocket 'write' messages and updates the redux store. */}
-
{/* The banner at the top of the page. */}