mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-13 19:33:46 +08:00
Ethan Weber authored and 
GitHub committed
1 parent
8af3a7a385
commit
cf34ef4bd0
26 files changed
+384
-355
No files matched your search
Vendored
+16
-16
@@ -27,10 +27,10 @@
|
||||
"justMyCode": true
|
||||
},
|
||||
{
|
||||
"name": "Python: run_train",
|
||||
"name": "Python: train",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"program": "scripts/run_train.py",
|
||||
"program": "scripts/train.py",
|
||||
"console": "integratedTerminal",
|
||||
"args": [
|
||||
"instant_ngp",
|
||||
@@ -39,20 +39,20 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Python: run_train ngp",
|
||||
"name": "Python: train ngp",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"program": "scripts/run_train.py",
|
||||
"program": "scripts/train.py",
|
||||
"console": "integratedTerminal",
|
||||
"args": [
|
||||
"instant_ngp"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Python: run_train ngp nerfactory",
|
||||
"name": "Python: train ngp nerfactory",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"program": "scripts/run_train.py",
|
||||
"program": "scripts/train.py",
|
||||
"console": "integratedTerminal",
|
||||
"args": [
|
||||
"instant_ngp",
|
||||
@@ -69,10 +69,10 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Python: run_train ngp-friends",
|
||||
"name": "Python: train ngp-friends",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"program": "scripts/run_train.py",
|
||||
"program": "scripts/train.py",
|
||||
"console": "integratedTerminal",
|
||||
"args": [
|
||||
"instant_ngp",
|
||||
@@ -88,40 +88,40 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Python: run_train mipnerf 360",
|
||||
"name": "Python: train mipnerf 360",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"program": "scripts/run_train.py",
|
||||
"program": "scripts/train.py",
|
||||
"console": "integratedTerminal",
|
||||
"args": [
|
||||
"mipnerf_360"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Python: run_train tensorf",
|
||||
"name": "Python: train tensorf",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"program": "scripts/run_train.py",
|
||||
"program": "scripts/train.py",
|
||||
"console": "integratedTerminal",
|
||||
"args": [
|
||||
"tensorf"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Python: run_train nerfw",
|
||||
"name": "Python: train nerfw",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"program": "scripts/run_train.py",
|
||||
"program": "scripts/train.py",
|
||||
"console": "integratedTerminal",
|
||||
"args": [
|
||||
"nerfw"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Python: run_train semantic_nerf",
|
||||
"name": "Python: train semantic_nerf",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"program": "scripts/run_train.py",
|
||||
"program": "scripts/train.py",
|
||||
"console": "integratedTerminal",
|
||||
"args": [
|
||||
"semantic_nerf"
|
||||
|
||||
@@ -134,13 +134,13 @@ To run with all the defaults, e.g. vanilla nerf method with the blender lego ima
|
||||
|
||||
```
|
||||
# To see what models are available.
|
||||
python scripts/run_train.py --help
|
||||
python scripts/train.py --help
|
||||
|
||||
# Run a vanilla nerf model.
|
||||
python scripts/run_train.py vanilla_nerf
|
||||
python scripts/train.py vanilla_nerf
|
||||
|
||||
# Run a faster version with instant ngp using tcnn (without the viewer).
|
||||
python scripts/run_train.py instant_ngp
|
||||
python scripts/train.py instant_ngp
|
||||
```
|
||||
|
||||
#### 3.x Training a model with the viewer
|
||||
@@ -149,16 +149,16 @@ Make sure to forward a port for the websocket to localhost. The default port is
|
||||
|
||||
```bash
|
||||
# with the default port
|
||||
python scripts/run_train.py instant_ngp --viewer.enable
|
||||
python scripts/train.py instant_ngp --viewer.enable
|
||||
|
||||
# with a specified websocket port
|
||||
python scripts/run_train.py instant_ngp --viewer.enable --viewer.websocket-port=7008
|
||||
python scripts/train.py instant_ngp --viewer.enable --viewer.websocket-port=7008
|
||||
|
||||
# with the viewer bridge server as a separate process
|
||||
# in one terminal, start the bridge server:
|
||||
viewer-bridge-server # or equivalently, python scripts/run_viewer_bridge_server.py
|
||||
# in another terminal, start training:
|
||||
python scripts/run_train.py instant_ngp --viewer.enable --viewer.no-launch-bridge-server
|
||||
python scripts/train.py instant_ngp --viewer.enable --viewer.no-launch-bridge-server
|
||||
```
|
||||
|
||||
#### 4. Visualizing training runs
|
||||
@@ -195,7 +195,7 @@ We support logging to weights and biases. To enable wandb logging, add the flag
|
||||
#### 5. Rendering a trajectories during inference
|
||||
|
||||
```
|
||||
python scripts/run_eval.py render-trajectory --load-config=outputs/blender_lego/instant_ngp/2022-07-07_230905/config.yml--traj=spiral --output-path=output.mp4
|
||||
python scripts/eval.py render-trajectory --load-config=outputs/blender_lego/instant_ngp/2022-07-07_230905/config.yml--traj=spiral --output-path=output.mp4
|
||||
```
|
||||
|
||||
#### 6. In-depth guide
|
||||
|
||||
@@ -7,7 +7,7 @@ We provide a set of pre implemented nerfactory models. One of the goals of nerfa
|
||||
It's easy!
|
||||
|
||||
```bash
|
||||
python scripts/run_train.py --config-name MODEL_CONFIG
|
||||
python scripts/train.py --config-name MODEL_CONFIG
|
||||
```
|
||||
|
||||
## Guides
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# Instant-NGP
|
||||
|
||||
<h4>Instant Neural Graphics Primitives with a Multiresolution Hash Encoding</h4>
|
||||
|
||||
```{button-link} https://nvlabs.github.io/instant-ngp/
|
||||
@@ -10,5 +11,5 @@ Paper Website
|
||||
### Running Model
|
||||
|
||||
```bash
|
||||
python scripts/run_train.py --config-name=graph_instant_ngp.yaml
|
||||
```
|
||||
python scripts/train.py --config-name=graph_instant_ngp.yaml
|
||||
```
|
||||
@@ -1,4 +1,5 @@
|
||||
# Mip-NeRF
|
||||
|
||||
<h4>A Multiscale Representation for Anti-Aliasing Neural Radiance Fields</h4>
|
||||
|
||||
```{button-link} https://jonbarron.info/mipnerf/
|
||||
@@ -10,7 +11,7 @@ Paper Website
|
||||
### Running Model
|
||||
|
||||
```bash
|
||||
python scripts/run_train.py --config-name=graph_mipnerf.yaml
|
||||
python scripts/train.py --config-name=graph_mipnerf.yaml
|
||||
```
|
||||
|
||||
## Overview
|
||||
@@ -39,4 +40,4 @@ The primary modification in MipNeRF is in the encoding for the field representat
|
||||
:width: 400
|
||||
```
|
||||
|
||||
In the field, the Positional Encoding (PE) is replaced with an Integrated Positional Encoding (IPE) that takes into account the size of the sample.
|
||||
In the field, the Positional Encoding (PE) is replaced with an Integrated Positional Encoding (IPE) that takes into account the size of the sample.
|
||||
@@ -1,4 +1,5 @@
|
||||
# Mip-NeRF 360
|
||||
|
||||
<h4>Unbounded Anti-Aliased Neural Radiance Fields</h4>
|
||||
|
||||
```{button-link} https://jonbarron.info/mipnerf360/
|
||||
@@ -10,5 +11,5 @@ Paper Website
|
||||
### Running Model
|
||||
|
||||
```bash
|
||||
python scripts/run_train.py --config-name=graph_mipnerf_360.yaml
|
||||
```
|
||||
python scripts/train.py --config-name=graph_mipnerf_360.yaml
|
||||
```
|
||||
@@ -11,7 +11,7 @@ Paper Website
|
||||
### Running the Model
|
||||
|
||||
```bash
|
||||
python scripts/run_train.py --config-name=graph_vanilla_nerf.yaml
|
||||
python scripts/train.py --config-name=graph_vanilla_nerf.yaml
|
||||
```
|
||||
|
||||
## Method
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# NeRF-W
|
||||
|
||||
<h4>NeRF in the Wild</h4>
|
||||
|
||||
```{button-link} https://nerf-w.github.io/
|
||||
@@ -10,5 +11,5 @@ Paper Website
|
||||
### Running Model
|
||||
|
||||
```bash
|
||||
python scripts/run_train.py --config-name=graph_nerf_w.yaml
|
||||
```
|
||||
python scripts/train.py --config-name=graph_nerf_w.yaml
|
||||
```
|
||||
@@ -3,5 +3,5 @@
|
||||
### Running Model
|
||||
|
||||
```bash
|
||||
python scripts/run_train.py --config-name=graph_semantic_nerf.yaml
|
||||
```
|
||||
python scripts/train.py --config-name=graph_semantic_nerf.yaml
|
||||
```
|
||||
@@ -1,7 +1,6 @@
|
||||
# Benchmarking workflow
|
||||
|
||||
|
||||
We make it easy to benchmark your new NeRF against the standard Blender dataset.
|
||||
We make it easy to benchmark your new NeRF against the standard Blender dataset.
|
||||
|
||||
## Launching training on Blender dataset
|
||||
|
||||
@@ -13,8 +12,9 @@ To launch training jobs automatically on each of these items, you can call:
|
||||
```
|
||||
|
||||
Simply replace the arguments in brackets with the correct arguments.
|
||||
* `-c`: name of the config you created for your new graph. For instance, if you have created the config `configs/graph_instant_ngp.yaml`, `<config-name>` will be `graph_instant_ngp`.
|
||||
* `-g`: specify the list of gpus you want to use on your machine space separated. for instance, if you want to use GPU's 0-3, you will need to pass in `0 1 2 3`. If left empty, the script will automatically find available GPU's and distribute training jobs on the available GPUs.
|
||||
|
||||
- `-c`: name of the config you created for your new graph. For instance, if you have created the config `configs/graph_instant_ngp.yaml`, `<config-name>` will be `graph_instant_ngp`.
|
||||
- `-g`: specify the list of gpus you want to use on your machine space separated. for instance, if you want to use GPU's 0-3, you will need to pass in `0 1 2 3`. If left empty, the script will automatically find available GPU's and distribute training jobs on the available GPUs.
|
||||
|
||||
A full example would be:
|
||||
|
||||
@@ -28,39 +28,42 @@ A full example would be:
|
||||
|
||||
The script will automatically launch training on all of the items and save the checkpoints in an output directory with the current date and timestamp.
|
||||
|
||||
|
||||
## Evaluating trained Blender models
|
||||
|
||||
Once you have launched training, and training converges, you can test your method with `scripts/benchmarking/launch_eval_blender.sh`.
|
||||
|
||||
Say we ran a benchmark on 08-10-2022 for instant_ngp. By default, the train script will save the benchmarks in the following format:
|
||||
|
||||
```
|
||||
outputs
|
||||
└───blender_chair_08-10-2022
|
||||
| └───instant_ngp
|
||||
| └───2022-08-10_172517
|
||||
| └───run_train.log
|
||||
| └───train.log
|
||||
| ...
|
||||
└───blender_drums_08-10-2022
|
||||
| └───instant_ngp
|
||||
| └───2022-08-10_172517
|
||||
| └───run_train.log
|
||||
| └───train.log
|
||||
| ...
|
||||
...
|
||||
```
|
||||
|
||||
If we wanted to run the benchmark on all the blender data for the above example, we would run:
|
||||
|
||||
```bash
|
||||
./scripts/benchmarking/launch_eval_blender.sh -c graph_instant_ngp -o ./outputs/ -m 08 -d 10 -y 2022 -s 172517 -g 4 5 6 7
|
||||
```
|
||||
|
||||
The flags used in the benchmarking script are defined as follows:
|
||||
* `-c`: config name (e.g. `graph_instant_ngp`). This should be the same as what was passed in for -c in the train script.
|
||||
* `-o`: base output directory for where all of the benchmarks are stored (e.g. `outputs/`). Corresponds to the hydra base dir.
|
||||
* `-m`: month of benchmark (e.g. `08`) of format 02d.
|
||||
* `-d`: date of benchmark (e.g. `10`) of format 02d.
|
||||
* `-y`: year of benchmark (e.g. `2022`).
|
||||
* `-s`: seconds timestamp of benchmark (e.g. `172517`) of format 06d.
|
||||
* `-g`: specifies the gpus to use and if not specified (no -g flag), will automaticaly search for available gpus.
|
||||
|
||||
- `-c`: config name (e.g. `graph_instant_ngp`). This should be the same as what was passed in for -c in the train script.
|
||||
- `-o`: base output directory for where all of the benchmarks are stored (e.g. `outputs/`). Corresponds to the hydra base dir.
|
||||
- `-m`: month of benchmark (e.g. `08`) of format 02d.
|
||||
- `-d`: date of benchmark (e.g. `10`) of format 02d.
|
||||
- `-y`: year of benchmark (e.g. `2022`).
|
||||
- `-s`: seconds timestamp of benchmark (e.g. `172517`) of format 06d.
|
||||
- `-g`: specifies the gpus to use and if not specified (no -g flag), will automaticaly search for available gpus.
|
||||
|
||||
The script will simultaneously run the benchmarking across all the objects in the blender dataset and calculates the PSNR/FPS/other stats. The results are saved as .json files in the `-o` directory with the following format:
|
||||
|
||||
@@ -70,4 +73,4 @@ outputs
|
||||
| └───blender_chair_08-10-2022_172517.json
|
||||
| | blender_ficus_08-10-2022_172517.json
|
||||
| | ...
|
||||
```
|
||||
```
|
||||
@@ -2,72 +2,77 @@
|
||||
|
||||
## Logging support
|
||||
|
||||
We provide integration with multiple logging interfaces to log images and statistics during training.
|
||||
We provide integration with multiple logging interfaces to log images and statistics during training.
|
||||
All of these loggers are configurable via the config:
|
||||
|
||||
```yaml
|
||||
# e.g. configs/graphs_default.yaml
|
||||
|
||||
logging:
|
||||
steps_per_log: 10 # how often to log losses/images
|
||||
max_buffer_size: 20 # defines how many steps to average over (if reporting averages)
|
||||
writer: # definition of all writers you want to use
|
||||
TensorboardWriter:
|
||||
log_dir: "./" # must specify the output path for all writers
|
||||
LocalWriter:
|
||||
log_dir: "./"
|
||||
stats_to_track:
|
||||
[ITER_TRAIN_TIME, TRAIN_RAYS_PER_SEC, CURR_TEST_PSNR]
|
||||
max_log_size: 10
|
||||
steps_per_log: 10 # how often to log losses/images
|
||||
max_buffer_size: 20 # defines how many steps to average over (if reporting averages)
|
||||
writer: # definition of all writers you want to use
|
||||
TensorboardWriter:
|
||||
log_dir: './' # must specify the output path for all writers
|
||||
LocalWriter:
|
||||
log_dir: './'
|
||||
stats_to_track: [ITER_TRAIN_TIME, TRAIN_RAYS_PER_SEC, CURR_TEST_PSNR]
|
||||
max_log_size: 10
|
||||
```
|
||||
|
||||
Currently, we provide support for the following loggers, which can be added to the `logging.writer` section of the config.
|
||||
|
||||
#### Tensorboard
|
||||
|
||||
To log training stats and images with [tensorboard](https://www.tensorflow.org/tensorboard):
|
||||
|
||||
```
|
||||
TensorboardWriter:
|
||||
log_dir: "./" # the output path for which the event files are written
|
||||
```
|
||||
|
||||
#### Weights and Biases
|
||||
|
||||
To log training stats and images with [wandb](https://wandb.ai/site):
|
||||
|
||||
```
|
||||
WandbWriter:
|
||||
log_dir: "./" # the output path for which the run files are written
|
||||
```
|
||||
|
||||
#### Local writer
|
||||
|
||||
```
|
||||
LocalWriter:
|
||||
log_dir: "./" # if images are saved, where the images would be saved
|
||||
stats_to_track:
|
||||
stats_to_track:
|
||||
[ITER_TRAIN_TIME, TRAIN_RAYS_PER_SEC, CURR_TEST_PSNR]
|
||||
max_log_size: 10
|
||||
max_log_size: 10
|
||||
```
|
||||
|
||||
The `LocalWriter` simply outputs to the terminal. However, you can specify additional parameters to customize the logging.
|
||||
* `stats_to_track`: all the stats that you want to print to the terminal (see list under `EventName` in `utils/writer.py`). You can add or remove any of the defined enums.
|
||||
* `max_log_size`: how much content to print onto the screen (e.g. only print 10 lines on the screen at a time). If 0, will print everything without deleting any previous lines.
|
||||
|
||||
- `stats_to_track`: all the stats that you want to print to the terminal (see list under `EventName` in `utils/writer.py`). You can add or remove any of the defined enums.
|
||||
- `max_log_size`: how much content to print onto the screen (e.g. only print 10 lines on the screen at a time). If 0, will print everything without deleting any previous lines.
|
||||
|
||||
## Code profiling support
|
||||
We provide built-in performance profiling capabilities to make it easier for you to debug and assess the performance of your code.
|
||||
|
||||
We provide built-in performance profiling capabilities to make it easier for you to debug and assess the performance of your code.
|
||||
|
||||
#### In-house profiler
|
||||
|
||||
You can use our built-in profiler by enabling the profiler in the config:
|
||||
|
||||
```yaml
|
||||
# e.g. configs/graphs_default.yaml
|
||||
|
||||
logging:
|
||||
enable_profiler: True
|
||||
enable_profiler: True
|
||||
```
|
||||
|
||||
The profiler computes the average total time of execution for any function with the `@profiler.time_function` decorator.
|
||||
The profiler computes the average total time of execution for any function with the `@profiler.time_function` decorator.
|
||||
For instance, if you wanted to profile the total time it takes to generate rays given pixel and camera indices via the `RayGenerator` class, you would want to evaluate its `forward()` function. In that class, you would need to add the decorator to the function.
|
||||
|
||||
|
||||
```python
|
||||
class RayGenerator(nn.Module):
|
||||
|
||||
@@ -79,12 +84,12 @@ class RayGenerator(nn.Module):
|
||||
...
|
||||
```
|
||||
|
||||
At termination of training or end of the training run, the profiler will print out the average execution time for all of the functions that have the profiler tag.
|
||||
At termination of training or end of the training run, the profiler will print out the average execution time for all of the functions that have the profiler tag.
|
||||
|
||||
Use this profiler if there are specific functions that you want to measure the times for.
|
||||
|
||||
|
||||
#### Profiling with PySpy
|
||||
|
||||
If you want to profile the entire codebase, we provide functionality to generate flame graphs or get real-time reports using [PySpy](https://github.com/benfred/py-spy).
|
||||
|
||||
Install PySpy
|
||||
@@ -102,11 +107,11 @@ To perform the profiling, you can either specify that you want to generate a fla
|
||||
|
||||
## for live view of functions
|
||||
|
||||
./scripts/debugging/profile.sh -t top -p <python_command>
|
||||
./scripts/debugging/profile.sh -t top -p <python_command>
|
||||
```
|
||||
|
||||
In both cases, `<python_command>` can be replaced with any command you would normally run in the terminal. For instance, you can replace `<python_function>` with `scripts/run_train.py data/dataset=blender_lego`. Then the full command would be e.g.:
|
||||
In both cases, `<python_command>` can be replaced with any command you would normally run in the terminal. For instance, you can replace `<python_function>` with `scripts/train.py data/dataset=blender_lego`. Then the full command would be e.g.:
|
||||
|
||||
```bash
|
||||
./scripts/debugging/profile.sh -t flame -o flame.svg -p scripts/run_train.py data/dataset=blender_lego
|
||||
```
|
||||
./scripts/debugging/profile.sh -t flame -o flame.svg -p scripts/train.py data/dataset=blender_lego
|
||||
```
|
||||
@@ -25,7 +25,7 @@ If your data was captured with a fisheye camera add `--camera-type fisheye`.
|
||||
Simply specify that you are using the `nerfactory` dataparser and point the data directory to your processed data.
|
||||
|
||||
```bash
|
||||
python scripts/run_train.py instant-ngp nerfactory --pipeline.datamanager.dataparser.data-directory PROCESSED_DATA_DIR
|
||||
python scripts/train.py instant-ngp nerfactory --pipeline.datamanager.dataparser.data-directory PROCESSED_DATA_DIR
|
||||
```
|
||||
|
||||
### Installing COLMAP
|
||||
|
||||
@@ -1,229 +1,229 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Developing Custom Models\n",
|
||||
"\n",
|
||||
"In this quick tour, we will walk you through the core of training and building any NeRFs with nerfactory.\n",
|
||||
"\n",
|
||||
"The entry point for training starts at `scripts/run_train.py`, which spawns instances of our `Trainer()` class (in `engine/trainer.py`). The `Trainer()` is responsible for setting up the datasets and NeRF graph depending on the config specified. It will then run the usual train/val routine for a config-specified number of iterations. If you are planning on using our codebase to build a new NeRF method or to use an existing implementation, we've abstracted away the training routine in these two files and chances are you will not need to think of them again.\n",
|
||||
"\n",
|
||||
"### Graphs, Modules, and Fields\n",
|
||||
"\n",
|
||||
"If you are looking to implemnet a new NeRF method or extend an existing one, you may need to set up a new graph (in `graphs/`).\n",
|
||||
"\n",
|
||||
"All NeRF graph definitions can be found in `graphs/`. The graph encapsulates all the modules in the NeRF method- from the MLP's to the samplers to the losses and optimizers. All graphs are composed of modules and fields.\n",
|
||||
"\n",
|
||||
"### Modules\n",
|
||||
"\n",
|
||||
"We can decompose all graphs into respective modules. For instance, we have {ref}`optimizers` which contains modules related to the loss, {ref}`renderers` which contains all rendering modules, {`graph/modules/`}, which includes samplers and ray generators. Additionally, we have {ref}`field_modules`, which will be discussed below.\n",
|
||||
"In general, modules can be thought of as individual component parts of the graph that can be swapped in/out. We provide most of the standard modules, but if you are creating a new NeRF graph, you may have to define new modules.\n",
|
||||
"\n",
|
||||
"### Fields\n",
|
||||
"\n",
|
||||
"{ref}`fields` represents the \"space\", aka. the radiance field of the NeRF. Here, we define the field as the part of the network that takes in point samples and any other conditioning, and outputs any of the `FieldHeadNames` (`nerf/field_modules/field_heads.py`).\n",
|
||||
"\n",
|
||||
"All fields are composed of modules continained within the `fields/` directory (e.g. `fields/modules/` or `fields/density_fields/`). We can think of field modules as modules that actually define or interact with the field.\n",
|
||||
"\n",
|
||||
"To build a NeRF field, we therefore follow these steps:\n",
|
||||
"\n",
|
||||
"1. define any relevant field modules\n",
|
||||
"2. create a new class that extends the {ref}`fields` base class and use these field modules to compose the full field\n",
|
||||
"3. implement the abstract functions `get_density()` and `get_outputs()`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "SyntaxError",
|
||||
"evalue": "invalid syntax (383175768.py, line 4)",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;36m Input \u001b[0;32mIn [1]\u001b[0;36m\u001b[0m\n\u001b[0;31m def __init__(self, ...) -> None:\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"class NeRFField(Field):\n",
|
||||
" \"\"\"NeRF Field\"\"\"\n",
|
||||
"\n",
|
||||
" def __init__(self, ...) -> None:\n",
|
||||
" \"\"\"Create boilerplate NeRF field.\"\"\"\n",
|
||||
"\n",
|
||||
" super().__init__()\n",
|
||||
"\n",
|
||||
" # In the init method, we are instantiating all relevant field modules\n",
|
||||
" self.position_encoding = position_encoding\n",
|
||||
" self.direction_encoding = direction_encoding\n",
|
||||
"\n",
|
||||
" self.mlp_base = MLP(\n",
|
||||
" in_dim=self.position_encoding.get_out_dim(),\n",
|
||||
" num_layers=base_mlp_num_layers,\n",
|
||||
" layer_width=base_mlp_layer_width,\n",
|
||||
" skip_connections=skip_connections,\n",
|
||||
" )\n",
|
||||
" ...\n",
|
||||
"\n",
|
||||
" def get_density(self, point_samples: PointSamples):\n",
|
||||
" \"\"\"\n",
|
||||
" Computes and returns the densities.\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" def get_outputs(\n",
|
||||
" self, point_samples: PointSamples, density_embedding: Optional[TensorType] = None\n",
|
||||
" ) -> Dict[FieldHeadNames, TensorType]:\n",
|
||||
" \"\"\"\n",
|
||||
" Computes and returns the outputs.\n",
|
||||
" \"\"\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Putting it together\n",
|
||||
"\n",
|
||||
"Now that we've gone over what modules and fields are, we are ready to create the graph!\n",
|
||||
"\n",
|
||||
"To implement the vanilla NeRF, we create a new class that inherits the abstract Graph class. Similar to composing a new field, we can pull from all pre-defined fields and modules and use them to compose a new Graph.\n",
|
||||
"We will then need to define the following abstract methods as seen in the skeleton code below. See also `graphs/vanilla_nerf.py` for the full implementation."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "NameError",
|
||||
"evalue": "name 'Graph' is not defined",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
|
||||
"\u001b[1;32m/shared/ethanweber/nerfactory/docs/tutorials/pipelines/creating_models.ipynb Cell 4\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2Bbirman/shared/ethanweber/nerfactory/docs/tutorials/pipelines/creating_models.ipynb#W6sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39mclass\u001b[39;00m \u001b[39mNeRFGraph\u001b[39;00m(Graph):\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bbirman/shared/ethanweber/nerfactory/docs/tutorials/pipelines/creating_models.ipynb#W6sdnNjb2RlLXJlbW90ZQ%3D%3D?line=1'>2</a>\u001b[0m \u001b[39m\"\"\"Vanilla NeRF graph\"\"\"\u001b[39;00m\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bbirman/shared/ethanweber/nerfactory/docs/tutorials/pipelines/creating_models.ipynb#W6sdnNjb2RlLXJlbW90ZQ%3D%3D?line=3'>4</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__init__\u001b[39m(\u001b[39mself\u001b[39m, intrinsics\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, camera_to_world\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mNone\u001b[39;00m:\n",
|
||||
"\u001b[0;31mNameError\u001b[0m: name 'Graph' is not defined"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"class NeRFGraph(Graph):\n",
|
||||
" \"\"\"Vanilla NeRF graph\"\"\"\n",
|
||||
"\n",
|
||||
" def __init__(self, intrinsics=None, camera_to_world=None, **kwargs) -> None:\n",
|
||||
" super().__init__(intrinsics=intrinsics, camera_to_world=camera_to_world, **kwargs)\n",
|
||||
"\n",
|
||||
" def populate_fields(self):\n",
|
||||
" \"\"\"\n",
|
||||
" Set all field related modules here\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" def populate_misc_modules(self):\n",
|
||||
" \"\"\"\n",
|
||||
" Set all remaining modules here including: samplers, renderers, losses, and metrics\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" def get_param_groups(self) -> Dict[str, List[Parameter]]:\n",
|
||||
" \"\"\"\n",
|
||||
" Create a dictionary of parameters that are grouped according to different optimizers\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" def get_outputs(self, ray_bundle: RayBundle):\n",
|
||||
" \"\"\"\n",
|
||||
" Takes in a Ray Bundle and returns a dictionary of outputs.\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" def get_loss_dict(self, outputs, batch):\n",
|
||||
" \"\"\"\n",
|
||||
" Computes and returns the losses.\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" def log_test_image_outputs(self, image_idx, step, batch, outputs):\n",
|
||||
" \"\"\"\n",
|
||||
" Writes the test image outputs.\n",
|
||||
" \"\"\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Creating the config\n",
|
||||
"\n",
|
||||
"Now that you have the graph and dataset all set up, you'll need to create the config that you pass into our run train routine. Our config system is powered by [Hydra](https://hydra.cc/). All Hydra and machine related arguments are stored in `configs/default_setup.yaml`, as well as the defaults list.\n",
|
||||
"To set up the graph config, create a new yaml under `configs/`.\n",
|
||||
"\n",
|
||||
"```yaml\n",
|
||||
"# configs/vanilla_nerf.yaml\n",
|
||||
"\n",
|
||||
"defaults:\n",
|
||||
" - graph_default # inherit the basic yaml heirarchy\n",
|
||||
" - _self_\n",
|
||||
"\n",
|
||||
"logging:\n",
|
||||
" # <insert logging definitions here>\n",
|
||||
"\n",
|
||||
"trainer:\n",
|
||||
" # <override configurations for loading/training/saving model here>\n",
|
||||
"\n",
|
||||
"experiment_name: blender_lego\n",
|
||||
"method_name: vanilla_nerf\n",
|
||||
"\n",
|
||||
"data:\n",
|
||||
" # <insert any dataset related overrides here>\n",
|
||||
"\n",
|
||||
"graph:\n",
|
||||
" _target_: nerfactory.graph.vanilla_nerf.NeRFGraph # set the target to the graph you defined\n",
|
||||
" # <insert any graph related overrides here>\n",
|
||||
"\n",
|
||||
"optimizers:\n",
|
||||
" # <insert any optimizer related overrides here>\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"### Training and testing\n",
|
||||
"\n",
|
||||
"Once you have the config properly set up, you can begin training! To begin the training process, just pass in the name of your new config to the training script:\n",
|
||||
"\n",
|
||||
"```bash\n",
|
||||
"python scripts/run_train.py --config-name vanilla_nerf\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"And now you have a brand-new NeRF in training! For testing and visualizing, simply refer to steps 4-5 in the [quickstart guide](https://github.com/plenoptix/nerfactory#quickstart).\n",
|
||||
"\n",
|
||||
"To help you get started, we also provide additional training tools such as profiling, logging, and debugging. Please refer to our [features guide](../../tooling/index.rst)."
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.8.13 ('nerfactory')",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.8.13"
|
||||
},
|
||||
"orig_nbformat": 4,
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "e691f033e0f2f1b9c0a11e7e81375a1e27aec87a71fd1b1eaa545f7a5e61b3f6"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Developing Custom Models\n",
|
||||
"\n",
|
||||
"In this quick tour, we will walk you through the core of training and building any NeRFs with nerfactory.\n",
|
||||
"\n",
|
||||
"The entry point for training starts at `scripts/train.py`, which spawns instances of our `Trainer()` class (in `engine/trainer.py`). The `Trainer()` is responsible for setting up the datasets and NeRF graph depending on the config specified. It will then run the usual train/val routine for a config-specified number of iterations. If you are planning on using our codebase to build a new NeRF method or to use an existing implementation, we've abstracted away the training routine in these two files and chances are you will not need to think of them again.\n",
|
||||
"\n",
|
||||
"### Graphs, Modules, and Fields\n",
|
||||
"\n",
|
||||
"If you are looking to implemnet a new NeRF method or extend an existing one, you may need to set up a new graph (in `graphs/`).\n",
|
||||
"\n",
|
||||
"All NeRF graph definitions can be found in `graphs/`. The graph encapsulates all the modules in the NeRF method- from the MLP's to the samplers to the losses and optimizers. All graphs are composed of modules and fields.\n",
|
||||
"\n",
|
||||
"### Modules\n",
|
||||
"\n",
|
||||
"We can decompose all graphs into respective modules. For instance, we have {ref}`optimizers` which contains modules related to the loss, {ref}`renderers` which contains all rendering modules, {`graph/modules/`}, which includes samplers and ray generators. Additionally, we have {ref}`field_modules`, which will be discussed below.\n",
|
||||
"In general, modules can be thought of as individual component parts of the graph that can be swapped in/out. We provide most of the standard modules, but if you are creating a new NeRF graph, you may have to define new modules.\n",
|
||||
"\n",
|
||||
"### Fields\n",
|
||||
"\n",
|
||||
"{ref}`fields` represents the \"space\", aka. the radiance field of the NeRF. Here, we define the field as the part of the network that takes in point samples and any other conditioning, and outputs any of the `FieldHeadNames` (`nerf/field_modules/field_heads.py`).\n",
|
||||
"\n",
|
||||
"All fields are composed of modules continained within the `fields/` directory (e.g. `fields/modules/` or `fields/density_fields/`). We can think of field modules as modules that actually define or interact with the field.\n",
|
||||
"\n",
|
||||
"To build a NeRF field, we therefore follow these steps:\n",
|
||||
"\n",
|
||||
"1. define any relevant field modules\n",
|
||||
"2. create a new class that extends the {ref}`fields` base class and use these field modules to compose the full field\n",
|
||||
"3. implement the abstract functions `get_density()` and `get_outputs()`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "SyntaxError",
|
||||
"evalue": "invalid syntax (383175768.py, line 4)",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;36m Input \u001b[0;32mIn [1]\u001b[0;36m\u001b[0m\n\u001b[0;31m def __init__(self, ...) -> None:\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"class NeRFField(Field):\n",
|
||||
" \"\"\"NeRF Field\"\"\"\n",
|
||||
"\n",
|
||||
" def __init__(self, ...) -> None:\n",
|
||||
" \"\"\"Create boilerplate NeRF field.\"\"\"\n",
|
||||
"\n",
|
||||
" super().__init__()\n",
|
||||
"\n",
|
||||
" # In the init method, we are instantiating all relevant field modules\n",
|
||||
" self.position_encoding = position_encoding\n",
|
||||
" self.direction_encoding = direction_encoding\n",
|
||||
"\n",
|
||||
" self.mlp_base = MLP(\n",
|
||||
" in_dim=self.position_encoding.get_out_dim(),\n",
|
||||
" num_layers=base_mlp_num_layers,\n",
|
||||
" layer_width=base_mlp_layer_width,\n",
|
||||
" skip_connections=skip_connections,\n",
|
||||
" )\n",
|
||||
" ...\n",
|
||||
"\n",
|
||||
" def get_density(self, point_samples: PointSamples):\n",
|
||||
" \"\"\"\n",
|
||||
" Computes and returns the densities.\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" def get_outputs(\n",
|
||||
" self, point_samples: PointSamples, density_embedding: Optional[TensorType] = None\n",
|
||||
" ) -> Dict[FieldHeadNames, TensorType]:\n",
|
||||
" \"\"\"\n",
|
||||
" Computes and returns the outputs.\n",
|
||||
" \"\"\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Putting it together\n",
|
||||
"\n",
|
||||
"Now that we've gone over what modules and fields are, we are ready to create the graph!\n",
|
||||
"\n",
|
||||
"To implement the vanilla NeRF, we create a new class that inherits the abstract Graph class. Similar to composing a new field, we can pull from all pre-defined fields and modules and use them to compose a new Graph.\n",
|
||||
"We will then need to define the following abstract methods as seen in the skeleton code below. See also `graphs/vanilla_nerf.py` for the full implementation."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "NameError",
|
||||
"evalue": "name 'Graph' is not defined",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
|
||||
"\u001b[1;32m/shared/ethanweber/nerfactory/docs/tutorials/pipelines/creating_models.ipynb Cell 4\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2Bbirman/shared/ethanweber/nerfactory/docs/tutorials/pipelines/creating_models.ipynb#W6sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39mclass\u001b[39;00m \u001b[39mNeRFGraph\u001b[39;00m(Graph):\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bbirman/shared/ethanweber/nerfactory/docs/tutorials/pipelines/creating_models.ipynb#W6sdnNjb2RlLXJlbW90ZQ%3D%3D?line=1'>2</a>\u001b[0m \u001b[39m\"\"\"Vanilla NeRF graph\"\"\"\u001b[39;00m\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bbirman/shared/ethanweber/nerfactory/docs/tutorials/pipelines/creating_models.ipynb#W6sdnNjb2RlLXJlbW90ZQ%3D%3D?line=3'>4</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__init__\u001b[39m(\u001b[39mself\u001b[39m, intrinsics\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, camera_to_world\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mNone\u001b[39;00m:\n",
|
||||
"\u001b[0;31mNameError\u001b[0m: name 'Graph' is not defined"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"class NeRFGraph(Graph):\n",
|
||||
" \"\"\"Vanilla NeRF graph\"\"\"\n",
|
||||
"\n",
|
||||
" def __init__(self, intrinsics=None, camera_to_world=None, **kwargs) -> None:\n",
|
||||
" super().__init__(intrinsics=intrinsics, camera_to_world=camera_to_world, **kwargs)\n",
|
||||
"\n",
|
||||
" def populate_fields(self):\n",
|
||||
" \"\"\"\n",
|
||||
" Set all field related modules here\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" def populate_misc_modules(self):\n",
|
||||
" \"\"\"\n",
|
||||
" Set all remaining modules here including: samplers, renderers, losses, and metrics\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" def get_param_groups(self) -> Dict[str, List[Parameter]]:\n",
|
||||
" \"\"\"\n",
|
||||
" Create a dictionary of parameters that are grouped according to different optimizers\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" def get_outputs(self, ray_bundle: RayBundle):\n",
|
||||
" \"\"\"\n",
|
||||
" Takes in a Ray Bundle and returns a dictionary of outputs.\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" def get_loss_dict(self, outputs, batch):\n",
|
||||
" \"\"\"\n",
|
||||
" Computes and returns the losses.\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" def log_test_image_outputs(self, image_idx, step, batch, outputs):\n",
|
||||
" \"\"\"\n",
|
||||
" Writes the test image outputs.\n",
|
||||
" \"\"\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Creating the config\n",
|
||||
"\n",
|
||||
"Now that you have the graph and dataset all set up, you'll need to create the config that you pass into our run train routine. Our config system is powered by [Hydra](https://hydra.cc/). All Hydra and machine related arguments are stored in `configs/default_setup.yaml`, as well as the defaults list.\n",
|
||||
"To set up the graph config, create a new yaml under `configs/`.\n",
|
||||
"\n",
|
||||
"```yaml\n",
|
||||
"# configs/vanilla_nerf.yaml\n",
|
||||
"\n",
|
||||
"defaults:\n",
|
||||
" - graph_default # inherit the basic yaml heirarchy\n",
|
||||
" - _self_\n",
|
||||
"\n",
|
||||
"logging:\n",
|
||||
" # <insert logging definitions here>\n",
|
||||
"\n",
|
||||
"trainer:\n",
|
||||
" # <override configurations for loading/training/saving model here>\n",
|
||||
"\n",
|
||||
"experiment_name: blender_lego\n",
|
||||
"method_name: vanilla_nerf\n",
|
||||
"\n",
|
||||
"data:\n",
|
||||
" # <insert any dataset related overrides here>\n",
|
||||
"\n",
|
||||
"graph:\n",
|
||||
" _target_: nerfactory.graph.vanilla_nerf.NeRFGraph # set the target to the graph you defined\n",
|
||||
" # <insert any graph related overrides here>\n",
|
||||
"\n",
|
||||
"optimizers:\n",
|
||||
" # <insert any optimizer related overrides here>\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"### Training and testing\n",
|
||||
"\n",
|
||||
"Once you have the config properly set up, you can begin training! To begin the training process, just pass in the name of your new config to the training script:\n",
|
||||
"\n",
|
||||
"```bash\n",
|
||||
"python scripts/train.py --config-name vanilla_nerf\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"And now you have a brand-new NeRF in training! For testing and visualizing, simply refer to steps 4-5 in the [quickstart guide](https://github.com/plenoptix/nerfactory#quickstart).\n",
|
||||
"\n",
|
||||
"To help you get started, we also provide additional training tools such as profiling, logging, and debugging. Please refer to our [features guide](../../tooling/index.rst)."
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.8.13 ('nerfactory')",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.8.13"
|
||||
},
|
||||
"orig_nbformat": 4,
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "e691f033e0f2f1b9c0a11e7e81375a1e27aec87a71fd1b1eaa545f7a5e61b3f6"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -68,19 +68,19 @@ To run with all the defaults, e.g. vanilla nerf method with the blender lego ima
|
||||
Run a vanilla nerf model.
|
||||
|
||||
```bash
|
||||
python scripts/run_train.py
|
||||
python scripts/train.py
|
||||
```
|
||||
|
||||
Run a faster version with instant ngp using tcnn (without the viewer).
|
||||
|
||||
```bash
|
||||
python scripts/run_train.py --config-name=graph_instant_ngp.yaml
|
||||
python scripts/train.py --config-name=graph_instant_ngp.yaml
|
||||
```
|
||||
|
||||
Run with the viewer. However, you'll have to start the viewer server first. (See [viewer docs](../tutorials/viewer/viewer_quickstart.md))
|
||||
|
||||
```bash
|
||||
python scripts/run_train.py --config-name=graph_instant_ngp.yaml viewer.enable=true
|
||||
python scripts/train.py --config-name=graph_instant_ngp.yaml viewer.enable=true
|
||||
```
|
||||
|
||||
With support for [Hydra](https://hydra.cc/), you can run with other configurations by changing appropriate configs defined in `configs/` or by setting flags via command-line arguments.
|
||||
@@ -100,7 +100,7 @@ tensorboard --logdir outputs
|
||||
To evaluate the trained NeRF, we provide an evaluation script that allows you to do benchmarking (see our [benchmarking workflow](../tooling/benchmarking.md)) or to render out the scene with a custom trajectory and save the output to a video.
|
||||
|
||||
```bash
|
||||
python scripts/run_eval.py render-trajectory --load-config=outputs/blender_lego/instant_ngp/2022-07-07_230905/config.yml --traj=spiral --output-path=output.mp4
|
||||
python scripts/eval.py render-trajectory --load-config=outputs/blender_lego/instant_ngp/2022-07-07_230905/config.yml --traj=spiral --output-path=output.mp4
|
||||
```
|
||||
|
||||
Please note, this quickstart allows you to preform everything in a headless manner. We also provide a web-based viewer that allows you to easily monitor training or render out trajectories. See our [viewer docs](../tutorials/viewer/viewer_quickstart.md) for more.
|
||||
@@ -142,7 +142,7 @@ python -m pip install --upgrade pip
|
||||
|
||||
#### Runtime errors, "len(sources) > 0".
|
||||
|
||||
When running `run_train.py `, an error occurs when installing cuda files in the backend code.
|
||||
When running `train.py `, an error occurs when installing cuda files in the backend code.
|
||||
|
||||
**Solution**:
|
||||
This is a problem with not being able to detect the correct CUDA version, and can be fixed by updating the CUDA path environment variables:
|
||||
@@ -151,4 +151,4 @@ This is a problem with not being able to detect the correct CUDA version, and ca
|
||||
export CUDA_HOME=/usr/local/cuda
|
||||
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
|
||||
export PATH=$PATH:$CUDA_HOME/bin
|
||||
```
|
||||
```
|
||||
@@ -2,12 +2,10 @@
|
||||
|
||||
The tutorial below shows you how to host the viewer yourself and how you can use it to visualize training.
|
||||
|
||||
|
||||
#### Connection between nerfactory and the Bridge Server
|
||||
|
||||
In the center, we have the Bridge Server, which facilitates the connection between nerfactory code and the Client App. This server runs on the same machine that you are using nerfactory. It has a TCP Request/Reply (REQ/REP) connection that nerfactory can connect to with the Viewer object (left). We use [ZeroMQ](https://zeromq.org/), an open-sourced messaging library, to implement this lightweight TCP connection. The Viewer class can send commands to the Bridge Server and receive replies. The Bridge Server will either dispatch commands to the Client App via a websocket or it will return information stored in the Bridge Server state.
|
||||
|
||||
|
||||
#### Connection between the Bridge Server and the Client App
|
||||
|
||||
The connection between the Bridge Server and the Client App works with WebSockets and WebRTC.
|
||||
@@ -16,10 +14,8 @@ The connection between the Bridge Server and the Client App works with WebSocket
|
||||
|
||||
- **WebRTC connection** - We use WebRTC to stream images being rendered from nerfactory. The websocket connection if first used to establish the WebRTC connection. Then, the Client App constantly publishes camera pose information to the Bridge Server and stores the camera information (intrinsics and extrinsics). This information is then queried from the nerfactory code, used to render an image with some Graph, and then the image is send over the TCP connection and dispatched via WebRTC to render the stream of images.
|
||||
|
||||
|
||||
## Getting started
|
||||
|
||||
|
||||
#### Running the Bridge Server
|
||||
|
||||
The viewer server runs on the same machine that you use for training. The training code will connect to the server with a lightweight TCP connection.
|
||||
@@ -35,7 +31,6 @@ It should print out something of the form:
|
||||
"ZMQWebSocketBridge using zmq_url=tcp://127.0.0.1:6000 and websocket_port=8051"
|
||||
```
|
||||
|
||||
|
||||
#### Running the Client App
|
||||
|
||||
> We will host the viewer online in the future, but for now we have to run it locally.
|
||||
@@ -67,6 +62,7 @@ yarn start
|
||||
```
|
||||
|
||||
Forward port and open in your local browser. The URL takes the following form, `http://localhost:<forwarded_react_port>?localhost:<forwarded_backend_tcp_port>`. Notice the "?" character.
|
||||
|
||||
- http://localhost:4000?localhost:8051
|
||||
|
||||
Possibly edit the port in `app/.env.development` file with the following to adjust the PORT, for example.
|
||||
@@ -79,18 +75,16 @@ PORT=4000
|
||||
ESLINT_NO_DEV_ERRORS=true
|
||||
```
|
||||
|
||||
|
||||
#### Running the nerfactory Code
|
||||
|
||||
You can now simply run a training job and visualize progress by enabling the viewer during training:
|
||||
|
||||
```bash
|
||||
python scripts/run_train.py --config-name=graph_instant_ngp.yaml viewer.enable=true
|
||||
python scripts/train.py --config-name=graph_instant_ngp.yaml viewer.enable=true
|
||||
```
|
||||
|
||||
- **Notebook demo** - See [Programming the viewer](viewer_notebook.ipynb) for an overview for how to interact with the viewer with the Viewer object from nerfactory.
|
||||
|
||||
|
||||
## Acknowledgements and references
|
||||
|
||||
We thank [Robin Deits](https://github.com/rdeits) and other contributors to the following repos, which we've started with and modified and extended for our use.
|
||||
@@ -106,4 +100,4 @@ Here are other resources that we've used and/or have found helpful while buildin
|
||||
|
||||
```
|
||||
|
||||
```
|
||||
```
|
||||
@@ -12,7 +12,7 @@ The viewer is built using [ThreeJS](https://threejs.org/) and packaged into a [R
|
||||
|
||||
2. Run training script with viewer enabled. Note that you should only use the viewer with a fast pipeline, such as instant ngp.
|
||||
```shell
|
||||
python scripts/run_train.py --config-name=graph_instant_ngp.yaml viewer.enable=True
|
||||
python scripts/train.py --config-name=graph_instant_ngp.yaml viewer.enable=True
|
||||
```
|
||||
|
||||
This pulls up a hosted web-viewer which is the fastest way to get started with visualizing your nerfactory training progress!
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "viewer",
|
||||
"homepage": ".",
|
||||
"version": "22-09-14-0",
|
||||
"version": "22-09-27-0",
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@emotion/react": "^11.10.4",
|
||||
|
||||
@@ -1,25 +1,6 @@
|
||||
import * as React from 'react';
|
||||
import * as THREE from 'three';
|
||||
|
||||
import { Button, Slider } from '@mui/material';
|
||||
import { MeshLine, MeshLineMaterial } from 'meshline';
|
||||
import { useContext, useEffect } from 'react';
|
||||
import { useDispatch, useSelector } from 'react-redux';
|
||||
|
||||
import ArrowBackIosNewIcon from '@mui/icons-material/ArrowBackIosNew';
|
||||
import ArrowForwardIosIcon from '@mui/icons-material/ArrowForwardIos';
|
||||
import ContentPasteGoIcon from '@mui/icons-material/ContentPasteGo';
|
||||
import DeleteIcon from '@mui/icons-material/Delete';
|
||||
import FirstPageIcon from '@mui/icons-material/FirstPage';
|
||||
import IconButton from '@mui/material/IconButton';
|
||||
import LastPageIcon from '@mui/icons-material/LastPage';
|
||||
import PauseIcon from '@mui/icons-material/Pause';
|
||||
import PlayArrowIcon from '@mui/icons-material/PlayArrow';
|
||||
import TextField from '@mui/material/TextField';
|
||||
import Tooltip from '@mui/material/Tooltip';
|
||||
import VisibilityIcon from '@mui/icons-material/Visibility';
|
||||
import { CSS2DObject } from 'three/examples/jsm/renderers/CSS2DRenderer';
|
||||
import { Stack } from '@mui/system';
|
||||
import {
|
||||
AllInclusiveOutlined,
|
||||
ChangeHistory,
|
||||
@@ -27,6 +8,25 @@ import {
|
||||
RadioButtonUnchecked,
|
||||
Replay,
|
||||
} from '@mui/icons-material';
|
||||
import { Button, Slider } from '@mui/material';
|
||||
import { MeshLine, MeshLineMaterial } from 'meshline';
|
||||
import { useContext, useEffect } from 'react';
|
||||
import { useDispatch, useSelector } from 'react-redux';
|
||||
|
||||
import ArrowBackIosNewIcon from '@mui/icons-material/ArrowBackIosNew';
|
||||
import ArrowForwardIosIcon from '@mui/icons-material/ArrowForwardIos';
|
||||
import { CSS2DObject } from 'three/examples/jsm/renderers/CSS2DRenderer';
|
||||
import ContentPasteGoIcon from '@mui/icons-material/ContentPasteGo';
|
||||
import DeleteIcon from '@mui/icons-material/Delete';
|
||||
import FirstPageIcon from '@mui/icons-material/FirstPage';
|
||||
import IconButton from '@mui/material/IconButton';
|
||||
import LastPageIcon from '@mui/icons-material/LastPage';
|
||||
import PauseIcon from '@mui/icons-material/Pause';
|
||||
import PlayArrowIcon from '@mui/icons-material/PlayArrow';
|
||||
import { Stack } from '@mui/system';
|
||||
import TextField from '@mui/material/TextField';
|
||||
import Tooltip from '@mui/material/Tooltip';
|
||||
import VisibilityIcon from '@mui/icons-material/Visibility';
|
||||
import { CameraHelper } from './CameraHelper';
|
||||
import { get_curve_object_from_cameras, get_transform_matrix } from './curve';
|
||||
import { WebSocketContext } from '../../WebSocket/WebSocket';
|
||||
@@ -452,7 +452,7 @@ export default function CameraPanel(props) {
|
||||
// Copy the text inside the text field
|
||||
const config_filename = `${config_base_dir}/config.yml`;
|
||||
const camera_path_filename = `${config_base_dir}/camera_path.json`;
|
||||
const cmd = `python scripts/run_eval.py render-trajectory --load-config ${config_filename} --traj filename --camera-path-filename ${camera_path_filename} --output-path output.mp4`;
|
||||
const cmd = `python scripts/eval.py render-trajectory --load-config ${config_filename} --traj filename --camera-path-filename ${camera_path_filename} --output-path output.mp4`;
|
||||
navigator.clipboard.writeText(cmd);
|
||||
|
||||
const camera_path_payload = {
|
||||
|
||||
+5
-2
@@ -74,7 +74,10 @@ docs = [
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
nf-scripts-viewer-bridge-server = "nerfactory.viewer.server.server:main"
|
||||
nf-install-cli = "scripts.completions.configure:entrypoint"
|
||||
nf-process-data = "scripts.process_data:entrypoint"
|
||||
nf-train = "scripts.train:entrypoint"
|
||||
nf-eval = "scripts.eval:entrypoint"
|
||||
|
||||
[options]
|
||||
# equivalent to using --extra-index-url with pip, which is needed for specifying the CUDA version torch and torchvision
|
||||
@@ -83,7 +86,7 @@ dependency_links = [
|
||||
]
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
include = ["nerfactory"]
|
||||
include = ["nerfactory", "scripts"]
|
||||
|
||||
# black
|
||||
[tool.black]
|
||||
|
||||
@@ -59,7 +59,7 @@ for dataset in ${DATASETS[@]}; do
|
||||
export CUDA_VISIBLE_DEVICES=${GPU_IDX[$idx]}
|
||||
base_config_name=${config_name/"graph_"/""}
|
||||
config_path="${output_dir}/blender_${dataset}_${month}-${date}-${year}/${base_config_name}/${year}-${month}-${date}_${seconds}/nerfactory_models/"
|
||||
python scripts/run_eval.py compute-psnr \
|
||||
python scripts/eval.py compute-psnr \
|
||||
--load-config=${config_path} \
|
||||
--output-path=${output_dir}/${base_config_name}/blender_${dataset}_${month}-${date}-${year}_${seconds}.json &
|
||||
echo "Launched ${checkpoint_dir} on gpu ${GPU_IDX[$idx]}"
|
||||
|
||||
@@ -52,7 +52,7 @@ len=${#GPU_IDX[@]}
|
||||
|
||||
for dataset in ${DATASETS[@]}; do
|
||||
export CUDA_VISIBLE_DEVICES=${GPU_IDX[$idx]}
|
||||
python scripts/run_train.py \
|
||||
python scripts/train.py \
|
||||
${config_name} \
|
||||
--logging.writer.2.no-enable \
|
||||
--pipeline.datamanager.train-dataset.data-directory=data/blender/${dataset} \
|
||||
|
||||
@@ -237,14 +237,19 @@ def main(
|
||||
CONSOLE.print()
|
||||
CONSOLE.print("Notes:")
|
||||
CONSOLE.print(
|
||||
":warning: In bash, completions will trigger via [dim white]./scripts/run_train.py <TAB>[/dim white], but"
|
||||
" not [dim white]python ./scripts/run_train.py <TAB>[/dim white]."
|
||||
":warning: In bash, completions will trigger via [dim white]./scripts/train.py <TAB>[/dim white], but"
|
||||
" not [dim white]python ./scripts/train.py <TAB>[/dim white]."
|
||||
)
|
||||
CONSOLE.print(
|
||||
":warning: Completions are in an experimental state. If you run into any issues, please file an issue!"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
def entrypoint():
|
||||
"""Entrypoint for use with pyproject scripts."""
|
||||
dcargs.extras.set_accent_color("bright_yellow")
|
||||
dcargs.cli(main, description=__doc__)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
entrypoint()
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
run_eval.py
|
||||
eval.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -197,7 +197,8 @@ class RenderTrajectory:
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
def entrypoint():
|
||||
"""Entrypoint for use with pyproject scripts."""
|
||||
# A Union over dataclass types will create a subcommand for each type.
|
||||
#
|
||||
# TODO: it would make sense to split this script up into separate scripts.
|
||||
@@ -208,3 +209,7 @@ if __name__ == "__main__":
|
||||
# checkpoint when resuming a training run.
|
||||
dcargs.extras.set_accent_color("bright_yellow")
|
||||
dcargs.cli(Union[ComputePSNR, RenderTrajectory]).main()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
entrypoint()
|
||||
@@ -482,6 +482,11 @@ def main(
|
||||
CONSOLE.rule()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
def entrypoint():
|
||||
"""Entrypoint for use with pyproject scripts."""
|
||||
dcargs.extras.set_accent_color("bright_yellow")
|
||||
dcargs.cli(main)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
entrypoint()
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
run_train_nerf.py
|
||||
train.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -204,7 +204,12 @@ def main(config: cfg.Config) -> None:
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
def entrypoint():
|
||||
"""Entrypoint for use with pyproject scripts."""
|
||||
# Choose a base configuration and override values.
|
||||
dcargs.extras.set_accent_color("bright_yellow")
|
||||
main(dcargs.cli(AnnotatedBaseConfigUnion))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
entrypoint()
|
||||
@@ -46,7 +46,7 @@ def set_reduced_config(config: Config):
|
||||
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::DeprecationWarning")
|
||||
def test_run_train():
|
||||
def test_train():
|
||||
"""test run train script works properly"""
|
||||
all_config_names = base_configs.keys()
|
||||
for config_name in all_config_names:
|
||||
@@ -61,4 +61,4 @@ def test_run_train():
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_run_train()
|
||||
test_train()
|
||||
Reference in new issue
Block a user