From 7ba68dde486a895f6142756b6257b93373701fae Mon Sep 17 00:00:00 2001 From: Ethan Weber Date: Mon, 3 Oct 2022 23:25:49 -0700 Subject: [PATCH] update pipeline docs (#643) update docs --- .../pipelines/datamanagers.ipynb | 34 --- .../pipelines/datamanagers.md | 90 +++++++ .../pipelines/dataparsers.ipynb | 158 ------------ .../developer_guides/pipelines/dataparsers.md | 154 ++++++++++++ docs/developer_guides/pipelines/fields.md | 5 + docs/developer_guides/pipelines/index.rst | 38 ++- docs/developer_guides/pipelines/models.ipynb | 229 ------------------ docs/developer_guides/pipelines/models.md | 144 +++++++++++ .../pipelines/pipelines.ipynb | 39 --- docs/developer_guides/pipelines/pipelines.md | 5 + .../data/dataparsers/base_dataparser.py | 10 +- .../data/dataparsers/nerfstudio_dataparser.py | 5 +- 12 files changed, 435 insertions(+), 476 deletions(-) delete mode 100644 docs/developer_guides/pipelines/datamanagers.ipynb create mode 100644 docs/developer_guides/pipelines/datamanagers.md delete mode 100644 docs/developer_guides/pipelines/dataparsers.ipynb create mode 100644 docs/developer_guides/pipelines/dataparsers.md create mode 100644 docs/developer_guides/pipelines/fields.md delete mode 100644 docs/developer_guides/pipelines/models.ipynb create mode 100644 docs/developer_guides/pipelines/models.md delete mode 100644 docs/developer_guides/pipelines/pipelines.ipynb create mode 100644 docs/developer_guides/pipelines/pipelines.md diff --git a/docs/developer_guides/pipelines/datamanagers.ipynb b/docs/developer_guides/pipelines/datamanagers.ipynb deleted file mode 100644 index a5bef2c61..000000000 --- a/docs/developer_guides/pipelines/datamanagers.ipynb +++ /dev/null @@ -1,34 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# DataManagers\n", - "\n", - "```{image} imgs/datamanagers.png\n", - ":align: center\n", - "```" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.8.13 ('nerfactory')", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.8.13" - }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "b7b86c17c0dfd6b313529972d424f2e075cc198a46cda005757126afe8133c7b" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs/developer_guides/pipelines/datamanagers.md b/docs/developer_guides/pipelines/datamanagers.md new file mode 100644 index 000000000..8aa29ca8c --- /dev/null +++ b/docs/developer_guides/pipelines/datamanagers.md @@ -0,0 +1,90 @@ +# DataManagers + +```{image} imgs/datamanagers.png +:align: center +``` + +## What is a DataManager? + +The DataManager returns RayBundle and RayGT objects. Let's first take a look at the most important abstract methods required by the DataManager. + +```python +class DataManager(nn.Module): + """Generic data manager's abstract class + """ + + @abstractmethod + def next_train(self, step: int) -> Tuple[RayBundle, Dict]: + """Returns the next batch of data for train.""" + + @abstractmethod + def next_eval(self, step: int) -> Tuple[RayBundle, Dict]: + """Returns the next batch of data for eval.""" + + @abstractmethod + def next_eval_image(self, step: int) -> Tuple[int, RayBundle, Dict]: + """Returns the next eval image. + + Returns: + The image index from the eval datset, the CameraRayBundle, and the RayGT dictionary. + """ +``` + +## Example + +We've implemented a VanillaDataManager that implements the standard logic of most NeRF papers. It will randomly sample training rays with corresponding ground truth information, in RayBundle and RayGT objects respectively. The config for the VanillaDataManager is the following. + +```python +@dataclass +class VanillaDataManagerConfig(InstantiateConfig): + """Configuration for data manager instantiation; DataManager is in charge of keeping the train/eval dataparsers; + After instantiation, data manager holds both train/eval datasets and is in charge of returning unpacked + train/eval data at each iteration + """ + + _target: Type = field(default_factory=lambda: VanillaDataManager) + """target class to instantiate""" + dataparser: AnnotatedDataParserUnion = BlenderDataParserConfig() + """specifies the dataparser used to unpack the data""" + train_num_rays_per_batch: int = 1024 + """number of rays per batch to use per training iteration""" + train_num_images_to_sample_from: int = -1 + """number of images to sample during training iteration""" + eval_num_rays_per_batch: int = 1024 + """number of rays per batch to use per eval iteration""" + eval_num_images_to_sample_from: int = -1 + """number of images to sample during eval iteration""" + eval_image_indices: Optional[Tuple[int, ...]] = (0,) + """specifies the image indices to use during eval; if None, uses all""" + train_camera_optimizer: CameraOptimizerConfig = CameraOptimizerConfig() + """specifies the camera pose optimizer used during training""" +``` + +Let's take a quick look at how the `run_train` method is implemented. Here we sample images, then pixels, and then return the RayBundle and RayGT information. + +```python +def next_train(self, step: int) -> Tuple[RayBundle, Dict]: + """Returns the next batch of data from the train dataloader.""" + self.train_count += 1 + # sample a batch of images + image_batch = next(self.iter_train_image_dataloader) + # sample pixels from this batch of images + batch = self.train_pixel_sampler.sample(image_batch) + ray_indices = batch["indices"] + # generate rays from this image and pixel indices + ray_bundle = self.train_ray_generator(ray_indices) + # return RayBundle and RayGT information + return ray_bundle, batch +``` + +You can see our code for more details. + +```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/datamanagers.py +:color: primary +:outline: +See the code! +``` + +## Creating Your Own + +We currently don't have other implementations because most papers follow the VanillaDataManager implementation. However, it should be straightforward to add a VanillaDataManager with logic that progressively adds cameras, for instance, by relying on the step and modifying RayBundle and RayGT generation logic. \ No newline at end of file diff --git a/docs/developer_guides/pipelines/dataparsers.ipynb b/docs/developer_guides/pipelines/dataparsers.ipynb deleted file mode 100644 index b4bdd85c6..000000000 --- a/docs/developer_guides/pipelines/dataparsers.ipynb +++ /dev/null @@ -1,158 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# DataParsers\n", - "\n", - "```{image} imgs/dataparsers.png\n", - ":align: center\n", - "```\n", - "\n", - "## What is a DataParser?\n", - "\n", - "The dataparser returns `DataparserOutputs`, which puts all the various datasets into a common format.\n", - "\n", - "```python\n", - "@dataclass\n", - "class DataparserOutputs:\n", - " \"\"\"Dataparser outputs for the image dataset and the ray generator.\"\"\"\n", - "\n", - " image_filenames: List[Path]\n", - " \"\"\"Filenames for the images.\"\"\"\n", - " cameras: Cameras\n", - " \"\"\"Camera object storing collection of camera information in dataset\"\"\"\n", - " alpha_color: Optional[TensorType[3]] = None\n", - " \"\"\"color of dataset background\"\"\"\n", - " scene_box: SceneBox = SceneBox()\n", - " \"\"\"scene box of dataset. Could be used to bound the scene, or to provide the scene scale depending on model.\"\"\"\n", - " ...\n", - "\n", - "@dataclass\n", - "class DataParser:\n", - "\n", - " @abstractmethod\n", - " def _generate_datapaser_outputs(self, split: str = \"train\") -> DataparserOutputs:\n", - " \"\"\"Abstract method that returns the dataparser outputs for the given split.\n", - "\n", - " Args:\n", - " split: Which dataset split to generate data for (train/eval/test).\n", - "\n", - " Returns:\n", - " DataparserOutputs for the specified dataset and split.\n", - " \"\"\"\n", - "```\n", - "\n", - "## Example\n", - "\n", - "Here is an example where we implement a DataParser for our Nerfstudio data format.\n", - "\n", - "```python\n", - "@dataclass\n", - "class NerfstudioDataParserConfig(DataParserConfig):\n", - " \"\"\"Nerfstudio dataset config\"\"\"\n", - "\n", - " _target: Type = field(default_factory=lambda: Nerfstudio)\n", - " \"\"\"Target class to instantiate.\"\"\"\n", - " data_directory: Path = Path(\"data/nerfstudio/poster\")\n", - " \"\"\"Directory specifying location of data.\"\"\"\n", - " scale_factor: float = 1.0\n", - " \"\"\"How much to scale the camera origins by.\"\"\"\n", - " downscale_factor: int = 1\n", - " \"\"\"How much to downscale images.\"\"\"\n", - " scene_scale: float = 1.0\n", - " \"\"\"How much to scale the region of interest by.\"\"\"\n", - " orientation_method: Literal[\"pca\", \"up\"] = \"up\"\n", - " \"\"\"The method to use for orientation.\"\"\"\n", - " train_split_percentage: float = 0.9\n", - " \"\"\"The percent of images to use for training. The remaining images are for eval.\n", - " \"\"\"\n", - "\n", - "@dataclass\n", - "class Nerfstudio(DataParser):\n", - " \"\"\"Nerfstudio Dataset\"\"\"\n", - "\n", - " config: NerfstudioDataParserConfig\n", - "\n", - " def _generate_datapaser_outputs(self, split=\"train\") -> DataParserOutputs:\n", - " \"\"\"Returns DataParserOutputs.\"\"\"\n", - "```\n", - "\n", - "## Our Implementations\n", - "\n", - "###### Nerfstudio\n", - "\n", - "This is our custom dataparser.\n", - "\n", - "```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/nerfstudio_dataparser.py\n", - ":color: primary\n", - ":outline:\n", - "See the code!\n", - "```\n", - "\n", - "###### Blender\n", - "\n", - "We support the synthetic Blender dataset from the original NeRF paper.\n", - "\n", - "```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/blender_dataparser.py\n", - ":color: primary\n", - ":outline:\n", - "See the code!\n", - "```\n", - "\n", - "###### Instant NGP\n", - "\n", - "This supports the Instant NGP dataset.\n", - "\n", - "```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/instant_ngp_dataparser.py\n", - ":color: primary\n", - ":outline:\n", - "See the code!\n", - "```\n", - "\n", - "###### MipNeRF 360\n", - "\n", - "```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/mipnerf_dataparser.py\n", - ":color: primary\n", - ":outline:\n", - "See the code!\n", - "```\n", - "\n", - "###### Record3D\n", - "\n", - "This dataparser can use recorded data from a >= iPhone 12 Pro using the Record3D app. Record a video and export with the `EXR + JPG sequence` format. Unzip export and `rgb` folder before training.\n", - "\n", - "```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/record3d_dataparser.py\n", - ":color: primary\n", - ":outline:\n", - "See the code!\n", - "```\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.8.12 ('nerfactory')", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.8.12" - }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "34c28001ff35fb390494047002768a8182dcf55b1b11415165e62ea61557ab83" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs/developer_guides/pipelines/dataparsers.md b/docs/developer_guides/pipelines/dataparsers.md new file mode 100644 index 000000000..046d456b3 --- /dev/null +++ b/docs/developer_guides/pipelines/dataparsers.md @@ -0,0 +1,154 @@ +# DataParsers + +```{image} imgs/dataparsers.png +:align: center +``` + +## What is a DataParser? + +The dataparser returns `DataparserOutputs`, which puts all the various datasets into a common format. The DataparserOutputs should be lightweight, containing filenames or other meta information which can later be processed by actual PyTorch Datasets and Dataloaders. The common format makes it easy to add another DataParser. All you have to do is implement the private method `_generate_dataparser_outputs` shown below. + +```python +@dataclass +class DataparserOutputs: + """Dataparser outputs for the which will be used by the DataManager + for creating RayBundle and RayGT objects.""" + + image_filenames: List[Path] + """Filenames for the images.""" + cameras: Cameras + """Camera object storing collection of camera information in dataset.""" + alpha_color: Optional[TensorType[3]] = None + """Color of dataset background.""" + scene_box: SceneBox = SceneBox() + """Scene box of dataset. Used to bound the scene or provide the scene scale depending on model.""" + semantics: Optional[Semantics] = None + """Semantics information.""" + +@dataclass +class DataParser: + + @abstractmethod + def _generate_dataparser_outputs(self, split: str = "train") -> DataparserOutputs: + """Abstract method that returns the dataparser outputs for the given split. + + Args: + split: Which dataset split to generate (train/test). + + Returns: + DataparserOutputs containing data for the specified dataset and split + """ +``` + +## Example + +Here is an example where we implement a DataParser for our Nerfstudio data format. + +```python +@dataclass +class NerfstudioDataParserConfig(DataParserConfig): + """Nerfstudio dataset config""" + + _target: Type = field(default_factory=lambda: Nerfstudio) + """target class to instantiate""" + data: Path = Path("data/nerfstudio/poster") + """Directory specifying location of data.""" + scale_factor: float = 1.0 + """How much to scale the camera origins by.""" + downscale_factor: Optional[int] = None + """How much to downscale images. If not set, images are chosen such that the max dimension is <1600px.""" + scene_scale: float = 1.0 + """How much to scale the region of interest by.""" + orientation_method: Literal["pca", "up"] = "up" + """The method to use for orientation.""" + train_split_percentage: float = 0.9 + """The percent of images to use for training. The remaining images are for eval.""" + +@dataclass +class Nerfstudio(DataParser): + """Nerfstudio DatasetParser""" + + config: NerfstudioDataParserConfig + + def _generate_dataparser_outputs(self, split="train"): + meta = load_from_json(self.config.data / "transforms.json") + image_filenames = [] + poses = [] + ... + dataparser_outputs = DataparserOutputs( + image_filenames=image_filenames, + cameras=cameras, + scene_box=scene_box, + ) + return dataparser_outputs +``` + +## Train and Eval Logic + +The DataParser will generate a train and eval DataparserOutputs depending on the `split` argument. For example, here is how you'd initialize some `InputDataset` classes that live in the DataManager. Because our DataparserOutputs maintain a common form, our Datasets should be plug-and-play. These datasets will load images needed to supervise the model with `RayGT` objects. + +```python +config = NerfstudioDataParserConfig() +dataparser = config.setup() +# train dataparser +dataparser_outputs = dataparser.get_dataparser_outputs(split="train") +input_dataset = InputDataset(dataparser_outputs) +``` + +You can also pull out information from the DataParserOutputs for other DataMangager componenets, such as the RayGenerator. The RayGenerator generates RayBundle objects from camera and pixel indices. + +```python +ray_generator = RayGenerator(dataparser_outputs.cameras) +``` + +## Our Implementations + +Below we enumerate the various dataparsers that we have implemented in our codebase. Feel free to use ours or add your own. Also any contributions are welcome and appreciated! + +###### Nerfstudio + +This is our custom dataparser. We have a script to convert images or videos with COLMAP to this format. + +```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/nerfstudio_dataparser.py +:color: primary +:outline: +See the code! +``` + +###### Blender + +We support the synthetic Blender dataset from the original NeRF paper. + +```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/blender_dataparser.py +:color: primary +:outline: +See the code! +``` + +###### Instant NGP + +This supports the Instant NGP dataset. + +```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/instant_ngp_dataparser.py +:color: primary +:outline: +See the code! +``` + +###### MipNeRF 360 + +```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/mipnerf_dataparser.py +:color: primary +:outline: +See the code! +``` + +###### Record3D + +This dataparser can use recorded data from a >= iPhone 12 Pro using the Record3D app. Record a video and export with the `EXR + JPG sequence` format. Unzip export and `rgb` folder before training. + +```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/data/dataparsers/record3d_dataparser.py +:color: primary +:outline: +See the code! +``` diff --git a/docs/developer_guides/pipelines/fields.md b/docs/developer_guides/pipelines/fields.md new file mode 100644 index 000000000..8f8108a50 --- /dev/null +++ b/docs/developer_guides/pipelines/fields.md @@ -0,0 +1,5 @@ +# Fields + +```{image} imgs/models.png +:align: center +``` \ No newline at end of file diff --git a/docs/developer_guides/pipelines/index.rst b/docs/developer_guides/pipelines/index.rst index 023f39068..863d6cf5a 100644 --- a/docs/developer_guides/pipelines/index.rst +++ b/docs/developer_guides/pipelines/index.rst @@ -1,19 +1,41 @@ Pipelines Overview ------------------------- -Here we show you how to create a custom pipeline. - -# TODO: show a figure of what a pipeline is here +Here we describe what a Pipeline is and how it works. You can see an overview figure with the major Pipeline components below. .. image:: imgs/pipelines_overview.png - :width: 400 :align: center + +.. admonition:: Note + + RayGT and RayOutputs are currently dictionaries. In the future, they will be typed objects. + + +Why Pipelines? + +Our goal is for any NeRF paper to be implemented as a Pipeline. + +The Pipeline is composed of two major components, namely the DataManager and the Model. The DataManager is responsible for loading data and generating RayBundle and RayGT objects. RayBundles are the input to the forward pass of the Model. These are needed for at both train and inference time. RayGT objects, however, are needed only during training to calculate the losses in the Loss Dict. + +RayBundle objects describe origins and viewing directions. The model will take these rays and render them into quanties as RayOutputs. RayGT contains the necessary ground truth (GT) information needed to compute losses. For example, the GT pixel values can be used to supervise the rendered rays with an L2 loss. + +In the following sections, we describe the Pipeline components and look at their code. + +.. toctree:: + :maxdepth: 1 + + dataparsers.md + datamanagers.md + models.md + fields.md + pipelines.md + +Implementing NeRF Papers + +Let's say you want to create a custom Pipeline that has a custom DataManager and a custom Model. Perhaps you care about dynmically adding cameras to the DataManager during training or you want to importance sample and generate rays from pixels where the loss is high. This can be accomplished by mixing and matching components into a Pipeline. The following guide will take you through an example of this. + .. toctree:: :maxdepth: 1 - dataparsers.ipynb - datamanagers.ipynb - models.ipynb - pipelines.ipynb creating_new_methods.ipynb \ No newline at end of file diff --git a/docs/developer_guides/pipelines/models.ipynb b/docs/developer_guides/pipelines/models.ipynb deleted file mode 100644 index b1dfbe57b..000000000 --- a/docs/developer_guides/pipelines/models.ipynb +++ /dev/null @@ -1,229 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Models\n", - "\n", - "```{image} imgs/models.png\n", - ":align: center\n", - "```\n", - "\n", - "In this quick tour, we will walk you through the core of training and building any NeRFs with nerfstudio.\n", - "\n", - "The entry point for training starts at `scripts/train.py`, which spawns instances of our `Trainer()` class (in `engine/trainer.py`). The `Trainer()` is responsible for setting up the datasets and NeRF graph depending on the config specified. It will then run the usual train/val routine for a config-specified number of iterations. If you are planning on using our codebase to build a new NeRF method or to use an existing implementation, we've abstracted away the training routine in these two files and chances are you will not need to think of them again.\n", - "\n", - "### Graphs, Modules, and Fields\n", - "\n", - "If you are looking to implemnet a new NeRF method or extend an existing one, you may need to set up a new graph (in `graphs/`).\n", - "\n", - "All NeRF graph definitions can be found in `graphs/`. The graph encapsulates all the modules in the NeRF method- from the MLP's to the samplers to the losses and optimizers. All graphs are composed of modules and fields.\n", - "\n", - "### Modules\n", - "\n", - "We can decompose all graphs into respective modules. For instance, we have `optimizers` which contains modules related to the loss, `renderers` which contains all rendering modules, {`graph/modules/`}, which includes samplers and ray generators. Additionally, we have `field_modules`, which will be discussed below.\n", - "In general, modules can be thought of as individual component parts of the graph that can be swapped in/out. We provide most of the standard modules, but if you are creating a new NeRF graph, you may have to define new modules.\n", - "\n", - "### Fields\n", - "\n", - "`fields` represents the \"space\", aka. the radiance field of the NeRF. Here, we define the field as the part of the network that takes in point samples and any other conditioning, and outputs any of the `FieldHeadNames` (`nerf/field_modules/field_heads.py`).\n", - "\n", - "All fields are composed of modules continained within the `fields/` directory (e.g. `fields/modules/`). We can think of field modules as modules that actually define or interact with the field.\n", - "\n", - "To build a NeRF field, we therefore follow these steps:\n", - "\n", - "1. define any relevant field modules\n", - "2. create a new class that extends the `fields` base class and use these field modules to compose the full field\n", - "3. implement the abstract functions `get_density()` and `get_outputs()`." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "ename": "SyntaxError", - "evalue": "invalid syntax (383175768.py, line 4)", - "output_type": "error", - "traceback": [ - "\u001b[0;36m Input \u001b[0;32mIn [1]\u001b[0;36m\u001b[0m\n\u001b[0;31m def __init__(self, ...) -> None:\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n" - ] - } - ], - "source": [ - "class NeRFField(Field):\n", - " \"\"\"NeRF Field\"\"\"\n", - "\n", - " def __init__(self, ...) -> None:\n", - " \"\"\"Create boilerplate NeRF field.\"\"\"\n", - "\n", - " super().__init__()\n", - "\n", - " # In the init method, we are instantiating all relevant field modules\n", - " self.position_encoding = position_encoding\n", - " self.direction_encoding = direction_encoding\n", - "\n", - " self.mlp_base = MLP(\n", - " in_dim=self.position_encoding.get_out_dim(),\n", - " num_layers=base_mlp_num_layers,\n", - " layer_width=base_mlp_layer_width,\n", - " skip_connections=skip_connections,\n", - " )\n", - " ...\n", - "\n", - " def get_density(self, point_samples: PointSamples):\n", - " \"\"\"\n", - " Computes and returns the densities.\n", - " \"\"\"\n", - "\n", - " def get_outputs(\n", - " self, point_samples: PointSamples, density_embedding: Optional[TensorType] = None\n", - " ) -> Dict[FieldHeadNames, TensorType]:\n", - " \"\"\"\n", - " Computes and returns the outputs.\n", - " \"\"\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Putting it together\n", - "\n", - "Now that we've gone over what modules and fields are, we are ready to create the graph!\n", - "\n", - "To implement the vanilla NeRF, we create a new class that inherits the abstract Graph class. Similar to composing a new field, we can pull from all pre-defined fields and modules and use them to compose a new Graph.\n", - "We will then need to define the following abstract methods as seen in the skeleton code below. See also `graphs/vanilla_nerf.py` for the full implementation." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "ename": "NameError", - "evalue": "name 'Graph' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m/shared/ethanweber/nerfactory/docs/tutorials/pipelines/creating_models.ipynb Cell 4\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[39mclass\u001b[39;00m \u001b[39mNeRFGraph\u001b[39;00m(Graph):\n\u001b[1;32m 2\u001b[0m \u001b[39m\"\"\"Vanilla NeRF graph\"\"\"\u001b[39;00m\n\u001b[1;32m 4\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__init__\u001b[39m(\u001b[39mself\u001b[39m, intrinsics\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, camera_to_world\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mNone\u001b[39;00m:\n", - "\u001b[0;31mNameError\u001b[0m: name 'Graph' is not defined" - ] - } - ], - "source": [ - "class NeRFGraph(Graph):\n", - " \"\"\"Vanilla NeRF graph\"\"\"\n", - "\n", - " def __init__(self, intrinsics=None, camera_to_world=None, **kwargs) -> None:\n", - " super().__init__(intrinsics=intrinsics, camera_to_world=camera_to_world, **kwargs)\n", - "\n", - " def populate_fields(self):\n", - " \"\"\"\n", - " Set all field related modules here\n", - " \"\"\"\n", - "\n", - " def populate_misc_modules(self):\n", - " \"\"\"\n", - " Set all remaining modules here including: samplers, renderers, losses, and metrics\n", - " \"\"\"\n", - "\n", - " def get_param_groups(self) -> Dict[str, List[Parameter]]:\n", - " \"\"\"\n", - " Create a dictionary of parameters that are grouped according to different optimizers\n", - " \"\"\"\n", - "\n", - " def get_outputs(self, ray_bundle: RayBundle):\n", - " \"\"\"\n", - " Takes in a Ray Bundle and returns a dictionary of outputs.\n", - " \"\"\"\n", - "\n", - " def get_loss_dict(self, outputs, batch):\n", - " \"\"\"\n", - " Computes and returns the losses.\n", - " \"\"\"\n", - "\n", - " def log_test_image_outputs(self, image_idx, step, batch, outputs):\n", - " \"\"\"\n", - " Writes the test image outputs.\n", - " \"\"\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Creating the config\n", - "\n", - "Now that you have the graph and dataset all set up, you'll need to create the config that you pass into our run train routine. Our config system is powered by [Hydra](https://hydra.cc/). All Hydra and machine related arguments are stored in `configs/default_setup.yaml`, as well as the defaults list.\n", - "To set up the graph config, create a new yaml under `configs/`.\n", - "\n", - "```yaml\n", - "# configs/vanilla_nerf.yaml\n", - "\n", - "defaults:\n", - " - graph_default # inherit the basic yaml heirarchy\n", - " - _self_\n", - "\n", - "logging:\n", - " # \n", - "\n", - "trainer:\n", - " # \n", - "\n", - "experiment_name: blender_lego\n", - "method_name: vanilla_nerf\n", - "\n", - "data:\n", - " # \n", - "\n", - "graph:\n", - " _target_: nerfstudio.graph.vanilla_nerf.NeRFGraph # set the target to the graph you defined\n", - " # \n", - "\n", - "optimizers:\n", - " # \n", - "```\n", - "\n", - "### Training and testing\n", - "\n", - "Once you have the config properly set up, you can begin training! To begin the training process, just pass in the name of your new config to the training script:\n", - "\n", - "```bash\n", - "python scripts/train.py --config-name vanilla_nerf\n", - "```" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.8.12 ('nerfactory')", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.12" - }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "34c28001ff35fb390494047002768a8182dcf55b1b11415165e62ea61557ab83" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs/developer_guides/pipelines/models.md b/docs/developer_guides/pipelines/models.md new file mode 100644 index 000000000..bdad496e1 --- /dev/null +++ b/docs/developer_guides/pipelines/models.md @@ -0,0 +1,144 @@ +# Models + +```{image} imgs/models.png +:align: center +``` + +## What is a Model? + +A Model is probably what you think of when you think of a NeRF paper. Often the phrases "Model" and "Method" are used interchangeably and for this reason, our [implemented models](https://docs.nerf.studio/en/latest/nerfology/models/index.html) are actually methods. However, most of the implementations details of a method happen in the model code, as you'll see in this section. + +A model, at a high level, takes in regions of space described by RayBundle objects, samples points along these rays, and returns rendered values for each ray. So, let's take a look at what it takes to create your own model! + +## Functions to Implement + +[The code](https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/models/base_model.py) is quite verbose, so here we distill the most important functions with succint descriptions. + +```python +class Model: + + config: ModelConfig + """Set the model config so that Python gives you typed autocomplete!""" + + def populate_modules(self): + """Set the fields and modules.""" + + # Fields + + # Ray Samplers + + # Colliders + + # Renderers + + # Losses + + # Metrics + + def get_param_groups(self) -> Dict[str, List[Parameter]]: + """Returns the parameter groups needed to optimizer your model components.""" + + def get_training_callbacks( + self, training_callback_attributes: TrainingCallbackAttributes + ) -> List[TrainingCallback]: + """Returns the training callbacks, such as updating a density grid for Instant NGP.""" + + def get_outputs(self, ray_bundle: RayBundle): + """Process a RayBundle object and return RayOutputs describing quanties for each ray.""" + + def get_metrics_dict(self, outputs, batch): + """Returns metrics dictionary which will be plotted with wandb or tensorboard.""" + + def get_loss_dict(self, outputs, batch, metrics_dict=None): + """Returns a dictionary of losses to be summed which will be your loss.""" + + def get_image_metrics_and_images( + self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor] + ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: + """Returns a dictionary of images and metrics to plot. Here you can apply your colormaps.""" +``` + +## Pythonic Configs with Models + +Our config system is most useful when it comes to models. Let's take a look at our Nerfacto model config. + +```python +@dataclass +class NerfactoModelConfig(ModelConfig): + """Nerfacto Model Config""" + + _target: Type = field(default_factory=lambda: NerfactoModel) + near_plane: float = 0.05 + """How far along the ray to start sampling.""" + far_plane: float = 1000.0 + """How far along the ray to stop sampling.""" + background_color: Literal["background", "last_sample"] = "last_sample" + """Whether to randomize the background color.""" + num_proposal_samples_per_ray: Tuple[int] = (64,) + """Number of samples per ray for the proposal network.""" + num_nerf_samples_per_ray: int = 64 + """Number of samples per ray for the nerf network.""" + num_proposal_network_iterations: int = 1 + """Number of proposal network iterations.""" + use_same_proposal_network: bool = False + """Use the same proposal network. Otherwise use different ones.""" + interlevel_loss_mult: float = 1.0 + """Proposal loss multiplier.""" + distortion_loss_mult: float = 0.002 + """Distortion loss multiplier.""" + use_proposal_weight_anneal: bool = True + """Whether to use proposal weight annealing.""" + use_average_appearance_embedding: bool = True + """Whether to use average appearance embedding or zeros for inference.""" + proposal_weights_anneal_slope: float = 10.0 + """Slope of the annealing function for the proposal weights.""" + proposal_weights_anneal_max_num_iters: int = 1000 + """Max num iterations for the annealing function.""" + use_single_jitter: bool = True + """Whether use single jitter or not for the proposal networks.""" +``` + +There are a lot of options! Thankfully, our config system makes this easy to handle. If you want to add another argument, you simply add a value to this config and when you type in `ns-train nerfacto --help`, it will show in the terminal as a value you can modify. + +Furthermore, you have Python autocomplete and static checking working in your favor. At the top of every Model, we specify the config and then can easily pull of values throughout the implementation. Let's take a look at the beginning of the NerfactoModel implementation. + +```python +class NerfactoModel(Model): + """Nerfacto model + + Args: + config: Nerfacto configuration to instantiate model + """ + + config: NerfactoModelConfig + + def populate_modules(self): + """Set the fields and modules.""" + ... + # Fields + self.field = TCNNNerfactoField( + self.scene_box.aabb, + spatial_distortion=scene_contraction, + num_images=self.num_train_data, + use_average_appearance_embedding=self.config.use_average_appearance_embedding, # notice self.config + ) + ... + # Renderers + self.renderer_rgb = RGBRenderer(background_color=self.config.background_color) # notice self.config + self.renderer_accumulation = AccumulationRenderer() + self.renderer_depth = DepthRenderer() +``` + +We invite you to take a look at the Nerfacto model and others to see how our models are formatted. + +```{button-link} https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/models/nerfacto.py +:color: primary +:outline: +See the code! +``` + +## Implementing a Model + +Now that you understand how the model is structured, you can create a model by populating these functions. We provide a library of model components to pull from when creating your model. Check out those tutorials here! + +One of these components is a Field, which you can learn more about in the next section. Fields associate a quantity of space with a value (e.g., density and color) and are used in every model. diff --git a/docs/developer_guides/pipelines/pipelines.ipynb b/docs/developer_guides/pipelines/pipelines.ipynb deleted file mode 100644 index ab79f9336..000000000 --- a/docs/developer_guides/pipelines/pipelines.ipynb +++ /dev/null @@ -1,39 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Pipelines\n", - "\n", - "```{image} imgs/pipelines.png\n", - ":align: center\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.8.13 ('nerfactory')", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.8.13" - }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "b7b86c17c0dfd6b313529972d424f2e075cc198a46cda005757126afe8133c7b" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/docs/developer_guides/pipelines/pipelines.md b/docs/developer_guides/pipelines/pipelines.md new file mode 100644 index 000000000..82c147a0d --- /dev/null +++ b/docs/developer_guides/pipelines/pipelines.md @@ -0,0 +1,5 @@ +# Pipelines + +```{image} imgs/pipelines.png +:align: center +``` \ No newline at end of file diff --git a/nerfstudio/data/dataparsers/base_dataparser.py b/nerfstudio/data/dataparsers/base_dataparser.py index 7b0e00a99..f671f17ea 100644 --- a/nerfstudio/data/dataparsers/base_dataparser.py +++ b/nerfstudio/data/dataparsers/base_dataparser.py @@ -50,19 +50,19 @@ class Semantics: @dataclass class DataparserOutputs: - """Dataparser outputs for the image dataset and the ray generator.""" + """Dataparser outputs for the which will be used by the DataManager + for creating RayBundle and RayGT objects.""" image_filenames: List[Path] """Filenames for the images.""" cameras: Cameras - """Camera object storing collection of camera information in dataset""" + """Camera object storing collection of camera information in dataset.""" alpha_color: Optional[TensorType[3]] = None - """color of dataset background""" + """Color of dataset background.""" scene_box: SceneBox = SceneBox() - """scene box of dataset. Could be used to bound the scene, or to provide the scene scale depending on model.""" + """Scene box of dataset. Used to bound the scene or provide the scene scale depending on model.""" semantics: Optional[Semantics] = None """Semantics information.""" - # we support additional input information/formats including mask/depth/semantics. additional_inputs: Dict[str, Any] = to_immutable_dict({}) """Dictionary of additional dataset information (e.g. semantics/point clouds/masks). {input_name: diff --git a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py index 29707e1a3..35f2412cf 100644 --- a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py +++ b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py @@ -58,13 +58,12 @@ class NerfstudioDataParserConfig(DataParserConfig): orientation_method: Literal["pca", "up"] = "up" """The method to use for orientation.""" train_split_percentage: float = 0.9 - """The percent of images to use for training. The remaining images are for eval. - """ + """The percent of images to use for training. The remaining images are for eval.""" @dataclass class Nerfstudio(DataParser): - """Nerfstudio Dataset""" + """Nerfstudio DatasetParser""" config: NerfstudioDataParserConfig downscale_factor: Optional[int] = None