diff --git a/.gitignore b/.gitignore index 0e88489f4..8a77cf87f 100644 --- a/.gitignore +++ b/.gitignore @@ -168,6 +168,7 @@ data data_*/ !pyrad/data !configs/data +!docs/tutorials/data # Misc old/ diff --git a/docs/conf.py b/docs/conf.py index 124a2b44d..56dcd2fd7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -42,6 +42,7 @@ extensions = [ "sphinx.ext.napoleon", "sphinx_copybutton", "nbsphinx", + "sphinx_design", ] # List of patterns, relative to source directory, that match files and @@ -61,6 +62,13 @@ html_title = "pyRad" autosectionlabel_prefix_document = True +html_theme_options = { + "light_css_variables": { + "color-brand-primary": "#00878e", + "color-brand-content": "#00878e", + }, +} + # -- Code block theme -------------------------------------------------------- pygments_style = "manni" diff --git a/docs/index.rst b/docs/index.rst index 78f377051..71e9039e7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,7 +25,7 @@ Contents :caption: Tutorials tutorials/quickstart.md - tutorials/creating_dataset.md + tutorials/data_setup.rst tutorials/creating_graphs.md Contributing diff --git a/docs/requirements.txt b/docs/requirements.txt index 2e1fe42a7..7f4b380c8 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -4,4 +4,5 @@ myst-parser==0.17.2 nbsphinx==0.8.9 sphinx==4.5.0 sphinxemoji==0.2.0 -sphinx-copybutton==0.5.0 \ No newline at end of file +sphinx-copybutton==0.5.0 +sphinx-design==0.2.0 \ No newline at end of file diff --git a/docs/tutorials/creating_dataset.md b/docs/tutorials/creating_dataset.md deleted file mode 100644 index 6d60c0d0b..000000000 --- a/docs/tutorials/creating_dataset.md +++ /dev/null @@ -1,96 +0,0 @@ -# Using custom data - -#### Running COLMAP on your own images - -Please download [this file](https://github.com/NVlabs/instant-ngp/blob/07d8e2ca7232f97397ab73af9b56c7db639d3445/scripts/colmap2nerf.py) and place it at `scripts/colmap2nerf.py` courtesy of the [Instant-NGP](https://github.com/NVlabs/instant-ngp) authors. This script will use [COLMAP](https://github.com/colmap/colmap) to process images of a video file into a reconstruction with camera poses. You can see their write-up [here](https://github.com/NVlabs/instant-ngp/blob/master/docs/nerf_dataset_tips.md) for instructions. - -For example, maybe we have a video named `bear.mp4`. We can use the commands described in the Instant-NGP repo to create our dataset. We first create a a data folder. Then, we navigate to this folder. Finally, we run the colmap2nerf.py script with a specified `video_fps` and `aabb_scale`. The recommended number of frames is around 50-150 and the `aabb_scale` is the extent of the scene. The origin is the center of the aabb and `aabb_scale` is the length of each side of the box. - -```bash -# Navigate to the repo. -cd /path/to/pyrad - -# Setup the environment variables. -PYRAD_DIR=`pwd` -DATASET_FORMAT=instant_ngp -DATASET_NAME=bear - -# Create and navigate to the data folder. -mkdir data/$DATASET_FORMAT/$DATASET_NAME -cd data/$DATASET_FORMAT/$DATASET_NAME - -# Run the COLMAP script. -python $PYRAD_DIR/scripts/colmap2nerf.py \ - --video_in /path/to/bear.mp4 \ - --video_fps 2 \ - --run_colmap \ - --aabb_scale 16 -``` - -###### Referring to a specific dataset - -Here is how we refer to a dataset. We have a `dataset_inputs_train` and a `dataset_inputs_eval` config. These live under the `data` directory. You can see `configs/graph_default.yaml` for more details on what a full config would look like. - -```yaml -data: - dataset_inputs_train: - data_directory: data/instant_ngp/fox - dataset_format: instant_ngp - downscale_factor: 1 - alpha_color: white - dataset_inputs_eval: - data_directory: data/instant_ngp/fox - dataset_format: instant_ngp - downscale_factor: 1 - alpha_color: white -``` - -#### Sanity check with a notebook - -You can use `notebooks/visualize_dataset.ipynb` to see how your formatted dataset looks. This code will load a dataset from a yaml config, show the camera poses, and visualize rays with the specified collider in the config. TODO: add a better visualization with images too - -![visualize_dataset](media/creating_dataset_visualize_dataset.png) - -#### Handling a new dataset type - -In this page, we explain how to use pyRad with your own data. We've implemented many common datasets inside the `pyrad/format/.py` files. Each of these files implements a function called `load__data`. Each of these functions returns a `DatasetInputs` object. Furthermore, `get_dataset_inputs` in `pyrad/data/utils.py` will have to be updated to handle the new function. - -```python -# The DatasetInputs dataclass that is passed around to create datasets and initialize the graphs. -# See `pyrad/data/structs.py` for the code. -@dataclass -class DatasetInputs: - """Dataset inputs are used to initialize datasets and the NeRF graph.""" - - image_filenames: List[str] - downscale_factor: int = 1 - intrinsics: torch.tensor = None - camera_to_world: torch.tensor = None - mask_filenames: List[str] = None - depth_filenames: List[str] = None - scene_bounds: SceneBounds = SceneBounds() - semantics: Semantics = Semantics() - point_cloud: PointCloud = PointCloud() - alpha_color: Optional[TensorType[3]] = None - -# The method signature for the `instant_ngp` dataset format. -# See `pyrad/data/format/instant_ngp.py` for the code. -def load_instant_ngp_data( - basedir: str, - downscale_factor: int = 1, - split: str = "train", camera_translation_scalar=0.33 -) -> DatasetInputs: - """Returns a DatasetInputs struct.""" - -# The method signature for get_dataset_inputs. -# Notice that the parameters match the yaml config parameters above. -# See `pyrad/data/utils.py` for the code. -def get_dataset_inputs( - data_directory: str, - dataset_format: str, - split: str, - downscale_factor: int = 1, - alpha_color: Optional[Union[str, list, ListConfig]] = None, -) -> DatasetInputs: - """Makes a call to `load__data` and returns a DatasetInputs struct.""" -``` diff --git a/docs/tutorials/data/choosing_dataset.md b/docs/tutorials/data/choosing_dataset.md new file mode 100644 index 000000000..145b3eb9c --- /dev/null +++ b/docs/tutorials/data/choosing_dataset.md @@ -0,0 +1,56 @@ +# Dataset configuration + +#### Referring to a specific dataset + +Here is how we refer to a dataset. We have a `dataset_inputs_train` and a `dataset_inputs_eval` config. These live under the `data` directory. You can see `configs/graph_default.yaml` for more details on what a full config would look like. + +```yaml +data: + dataset_inputs_train: + data_directory: data/instant_ngp/fox + dataset_format: instant_ngp + downscale_factor: 1 + alpha_color: white + dataset_inputs_eval: + data_directory: data/instant_ngp/fox + dataset_format: instant_ngp + downscale_factor: 1 + alpha_color: white +``` + +#### Sanity check with a notebook + +You can use `notebooks/visualize_dataset.ipynb` to see how your formatted dataset looks. This code will load a dataset from a yaml config, show the camera poses, and visualize rays with the specified collider in the config. TODO: add a better visualization with images too + +![visualize_dataset](media/creating_dataset_visualize_dataset.png) + + +#### Supported datasets + +We include support for a set of standard dataset formats. + +###### Blender Synthetic + +```{button-link} https://drive.google.com/drive/u/1/folders/128yBriW1IG_3NJ5Rp7APSTZsJqdJdfc1 +:color: primary +:outline: +Download +``` + +```yaml +dataset_format: blender +``` + +Classic NeRF dataset including the "lego bulldozer" scene that was released with the original NeRF paper. + +###### Instant NGP + +```{button-link} https://github.com/NVlabs/instant-ngp#nerf-fox +:color: primary +:outline: +Download +``` + +```yaml +dataset_format: instant_ngp +``` \ No newline at end of file diff --git a/docs/tutorials/data/creating_dataset.md b/docs/tutorials/data/creating_dataset.md new file mode 100644 index 000000000..5f6d83dcd --- /dev/null +++ b/docs/tutorials/data/creating_dataset.md @@ -0,0 +1,33 @@ +# Using custom data + +#### Running COLMAP on your own images + +Please download [this file](https://github.com/NVlabs/instant-ngp/blob/07d8e2ca7232f97397ab73af9b56c7db639d3445/scripts/colmap2nerf.py) and place it at `scripts/colmap2nerf.py` courtesy of the [Instant-NGP](https://github.com/NVlabs/instant-ngp) authors. This script will use [COLMAP](https://github.com/colmap/colmap) to process images of a video file into a reconstruction with camera poses. You can see their write-up [here](https://github.com/NVlabs/instant-ngp/blob/master/docs/nerf_dataset_tips.md) for instructions. + +For example, maybe we have a video named `bear.mp4`. We can use the commands described in the Instant-NGP repo to create our dataset. We first create a a data folder. Then, we navigate to this folder. Finally, we run the colmap2nerf.py script with a specified `video_fps` and `aabb_scale`. The recommended number of frames is around 50-150 and the `aabb_scale` is the extent of the scene. The origin is the center of the aabb and `aabb_scale` is the length of each side of the box. + +```bash +# Navigate to the repo. +cd /path/to/pyrad + +# Setup the environment variables. +PYRAD_DIR=`pwd` +DATASET_FORMAT=instant_ngp +DATASET_NAME=bear + +# Create and navigate to the data folder. +mkdir data/$DATASET_FORMAT/$DATASET_NAME +cd data/$DATASET_FORMAT/$DATASET_NAME + +# Run the COLMAP script. +python $PYRAD_DIR/scripts/colmap2nerf.py \ + --video_in /path/to/bear.mp4 \ + --video_fps 2 \ + --run_colmap \ + --aabb_scale 16 +``` + +In the config set: +```yaml +dataset_format: instant_ngp +``` \ No newline at end of file diff --git a/docs/tutorials/data/custom_dataset_format.md b/docs/tutorials/data/custom_dataset_format.md new file mode 100644 index 000000000..45a8c281e --- /dev/null +++ b/docs/tutorials/data/custom_dataset_format.md @@ -0,0 +1,44 @@ + +# Custom Data Formats + +In this page, we explain how to use pyRad with your own data. We've implemented many common datasets inside the `pyrad/format/.py` files. Each of these files implements a function called `load__data`. Each of these functions returns a `DatasetInputs` object. Furthermore, `get_dataset_inputs` in `pyrad/data/utils.py` will have to be updated to handle the new function. + +```python +# The DatasetInputs dataclass that is passed around to create datasets and initialize the graphs. +# See `pyrad/data/structs.py` for the code. +@dataclass +class DatasetInputs: + """Dataset inputs are used to initialize datasets and the NeRF graph.""" + + image_filenames: List[str] + downscale_factor: int = 1 + intrinsics: torch.tensor = None + camera_to_world: torch.tensor = None + mask_filenames: List[str] = None + depth_filenames: List[str] = None + scene_bounds: SceneBounds = SceneBounds() + semantics: Semantics = Semantics() + point_cloud: PointCloud = PointCloud() + alpha_color: Optional[TensorType[3]] = None + +# The method signature for the `instant_ngp` dataset format. +# See `pyrad/data/format/instant_ngp.py` for the code. +def load_instant_ngp_data( + basedir: str, + downscale_factor: int = 1, + split: str = "train", camera_translation_scalar=0.33 +) -> DatasetInputs: + """Returns a DatasetInputs struct.""" + +# The method signature for get_dataset_inputs. +# Notice that the parameters match the yaml config parameters above. +# See `pyrad/data/utils.py` for the code. +def get_dataset_inputs( + data_directory: str, + dataset_format: str, + split: str, + downscale_factor: int = 1, + alpha_color: Optional[Union[str, list, ListConfig]] = None, +) -> DatasetInputs: + """Makes a call to `load__data` and returns a DatasetInputs struct.""" +``` diff --git a/docs/tutorials/media/creating_dataset_visualize_dataset.png b/docs/tutorials/data/media/creating_dataset_visualize_dataset.png similarity index 100% rename from docs/tutorials/media/creating_dataset_visualize_dataset.png rename to docs/tutorials/data/media/creating_dataset_visualize_dataset.png diff --git a/docs/tutorials/data_setup.rst b/docs/tutorials/data_setup.rst new file mode 100644 index 000000000..676aa70f5 --- /dev/null +++ b/docs/tutorials/data_setup.rst @@ -0,0 +1,9 @@ + +Data Setup +----------- + +.. toctree:: + + data/choosing_dataset.md + data/creating_dataset.md + data/custom_dataset_format.md \ No newline at end of file diff --git a/docs/tutorials/quickstart.md b/docs/tutorials/quickstart.md index eff453053..18cfaa9cc 100644 --- a/docs/tutorials/quickstart.md +++ b/docs/tutorials/quickstart.md @@ -1,5 +1,12 @@ # Quickstart guide +```{include} ../../README.md +:relative-docs: docs/ +:relative-images: +:start-after: quick_tour.html). +:end-before: 6. In-depth +``` + These quickstart steps will help you get started with training and testing the default vanilla nerf on the classic blender lego scene.