diff --git a/.gitignore b/.gitignore index afd28f97a..31dafbea1 100644 --- a/.gitignore +++ b/.gitignore @@ -171,6 +171,7 @@ data data_*/ !nerfactory/data !configs/data +configs_custom !docs/tutorials/data # Misc diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index eeb5276d2..ae573a003 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -8,7 +8,8 @@ "defines": [], "compilerPath": "/usr/bin/gcc", "cStandard": "c11", - "intelliSenseMode": "linux-gcc-x64" + "intelliSenseMode": "linux-gcc-x64", + "configurationProvider": "ms-vscode.makefile-tools" } ], "version": 4 diff --git a/docs/dataloader_components/incrementally_add_cameras.ipynb b/docs/dataloader_components/incrementally_add_cameras.ipynb new file mode 100644 index 000000000..723669915 --- /dev/null +++ b/docs/dataloader_components/incrementally_add_cameras.ipynb @@ -0,0 +1,45 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Incrementally Add Cameras" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8.13 ('nerfactory')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.13" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "e691f033e0f2f1b9c0a11e7e81375a1e27aec87a71fd1b1eaa545f7a5e61b3f6" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/dataloader_components/index.md b/docs/dataloader_components/index.md new file mode 100644 index 000000000..2a1dd2ccf --- /dev/null +++ b/docs/dataloader_components/index.md @@ -0,0 +1,9 @@ +# Dataloader Components + +Here we show how flexible our Dataloader can be with a series of dataloading guides. + +```{toctree} + :maxdepth: 1 + Pixel Sampling Stategies + Incrementally Add Cameras +``` diff --git a/docs/dataloader_components/pixel_sampling_strategies.ipynb b/docs/dataloader_components/pixel_sampling_strategies.ipynb new file mode 100644 index 000000000..81fd5e01f --- /dev/null +++ b/docs/dataloader_components/pixel_sampling_strategies.ipynb @@ -0,0 +1,37 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Pixel Sampling Strategies" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8.13 ('nerfactory')", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.8.13" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "e691f033e0f2f1b9c0a11e7e81375a1e27aec87a71fd1b1eaa545f7a5e61b3f6" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/index.rst b/docs/index.rst index 0e3d30e3a..3a6132fc5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -37,17 +37,17 @@ Contents :caption: Getting Started Quickstart - tutorials/data/data_index - tutorials/models/creating_pipelines - tutorials/viewer/using_viewer + tutorials/data/index + tutorials/pipelines/index + tutorials/viewer/index .. toctree:: :hidden: :caption: Guides - models/index - notebooks/index + model_components/index + dataloader_components/index .. toctree:: :hidden: diff --git a/docs/notebooks/imgs/frustums-dark.png b/docs/model_components/imgs/frustums-dark.png similarity index 100% rename from docs/notebooks/imgs/frustums-dark.png rename to docs/model_components/imgs/frustums-dark.png diff --git a/docs/notebooks/imgs/frustums.png b/docs/model_components/imgs/frustums.png similarity index 100% rename from docs/notebooks/imgs/frustums.png rename to docs/model_components/imgs/frustums.png diff --git a/docs/notebooks/imgs/samplers_stratified-dark.png b/docs/model_components/imgs/samplers_stratified-dark.png similarity index 100% rename from docs/notebooks/imgs/samplers_stratified-dark.png rename to docs/model_components/imgs/samplers_stratified-dark.png diff --git a/docs/notebooks/imgs/samplers_stratified-light.png b/docs/model_components/imgs/samplers_stratified-light.png similarity index 100% rename from docs/notebooks/imgs/samplers_stratified-light.png rename to docs/model_components/imgs/samplers_stratified-light.png diff --git a/docs/notebooks/imgs/samplers_type-dark.png b/docs/model_components/imgs/samplers_type-dark.png similarity index 100% rename from docs/notebooks/imgs/samplers_type-dark.png rename to docs/model_components/imgs/samplers_type-dark.png diff --git a/docs/notebooks/imgs/samplers_type-light.png b/docs/model_components/imgs/samplers_type-light.png similarity index 100% rename from docs/notebooks/imgs/samplers_type-light.png rename to docs/model_components/imgs/samplers_type-light.png diff --git a/docs/notebooks/index.md b/docs/model_components/index.md similarity index 51% rename from docs/notebooks/index.md rename to docs/model_components/index.md index e9877a769..b64903820 100644 --- a/docs/notebooks/index.md +++ b/docs/model_components/index.md @@ -1,6 +1,6 @@ -# Components +# Model Components -It can be difficult getting started with NeRFs. The field is still quite new and most of the key nuggets are burried in research papers. For this reason we can consoladated many of the key concepts into a series of guides. +It can be difficult getting started with NeRFs. The reserach field is still quite new and most of the key nuggets are burried in academic papers. For this reason, we have consoladated many of the key concepts into a series of guides. ```{toctree} :maxdepth: 1 @@ -9,4 +9,4 @@ It can be difficult getting started with NeRFs. The field is still quite new and Ray Samplers Spatial Distortions Encoders -``` \ No newline at end of file +``` diff --git a/docs/notebooks/visualize_cameras.ipynb b/docs/model_components/visualize_cameras.ipynb similarity index 100% rename from docs/notebooks/visualize_cameras.ipynb rename to docs/model_components/visualize_cameras.ipynb diff --git a/docs/notebooks/visualize_encoders.ipynb b/docs/model_components/visualize_encoders.ipynb similarity index 100% rename from docs/notebooks/visualize_encoders.ipynb rename to docs/model_components/visualize_encoders.ipynb diff --git a/docs/notebooks/visualize_samplers.ipynb b/docs/model_components/visualize_samplers.ipynb similarity index 100% rename from docs/notebooks/visualize_samplers.ipynb rename to docs/model_components/visualize_samplers.ipynb diff --git a/docs/notebooks/visualize_samples.ipynb b/docs/model_components/visualize_samples.ipynb similarity index 100% rename from docs/notebooks/visualize_samples.ipynb rename to docs/model_components/visualize_samples.ipynb diff --git a/docs/notebooks/visualize_spatial_distortions.ipynb b/docs/model_components/visualize_spatial_distortions.ipynb similarity index 100% rename from docs/notebooks/visualize_spatial_distortions.ipynb rename to docs/model_components/visualize_spatial_distortions.ipynb diff --git a/docs/models/index.md b/docs/models/index.md index 786d16c14..0cf680014 100644 --- a/docs/models/index.md +++ b/docs/models/index.md @@ -19,7 +19,7 @@ In addition to their implementations, we have provided guides that walk through NeRF Mip-NeRF Mip-NeRF 360 - Instan-NGP + Instant-NGP NeRF-W Semantic NeRF ``` diff --git a/docs/models/nerf.md b/docs/models/nerf.md index 8f11b1922..cda7f01f5 100644 --- a/docs/models/nerf.md +++ b/docs/models/nerf.md @@ -50,7 +50,7 @@ NeRFs are a volumetric representation encoded into a neural network. They are no #### Positional Encoding -An extra trick is necessary to making the neural network expressive enough to represent fine details in the scene. The input coordinates $(x,y,z,\theta,\phi)$ need to be encoded to a higher dimensional space prior to being input into the network. You can learn more about encodings [here](../notebooks/visualize_encoders.ipynb). +An extra trick is necessary to making the neural network expressive enough to represent fine details in the scene. The input coordinates $(x,y,z,\theta,\phi)$ need to be encoded to a higher dimensional space prior to being input into the network. You can learn more about encodings [here](../model_components/visualize_encoders.ipynb). ### Rendering diff --git a/docs/tutorials/data/data_index.rst b/docs/tutorials/data/index.rst similarity index 100% rename from docs/tutorials/data/data_index.rst rename to docs/tutorials/data/index.rst diff --git a/docs/tutorials/models/creating_graphs.md b/docs/tutorials/models/creating_graphs.md deleted file mode 100644 index 1bfc9a688..000000000 --- a/docs/tutorials/models/creating_graphs.md +++ /dev/null @@ -1,151 +0,0 @@ -# Developing a Custom Model - -In this quick tour, we will walk you through the core of training and building any NeRFs with nerfactory. - -The entry point for training starts at `scripts/run_train.py`, which spawns instances of our `Trainer()` class (in `engine/trainer.py`). The `Trainer()` is responsible for setting up the datasets and NeRF graph depending on the config specified. It will then run the usual train/val routine for a config-specified number of iterations. If you are planning on using our codebase to build a new NeRF method or to use an existing implementation, we've abstracted away the training routine in these two files and chances are you will not need to think of them again. - -## Graphs, Modules, and Fields - -If you are looking to implemnet a new NeRF method or extend an existing one, you may need to set up a new graph (in `graphs/`). - -All NeRF graph definitions can be found in `graphs/`. The graph encapsulates all the modules in the NeRF method- from the MLP's to the samplers to the losses and optimizers. All graphs are composed of modules and fields. - -### Modules - -We can decompose all graphs into respective modules. For instance, we have {ref}`optimizers` which contains modules related to the loss, {ref}`renderers` which contains all rendering modules, {`graph/modules/`}, which includes samplers and ray generators. Additionally, we have {ref}`field_modules`, which will be discussed below. -In general, modules can be thought of as individual component parts of the graph that can be swapped in/out. We provide most of the standard modules, but if you are creating a new NeRF graph, you may have to define new modules. - -### Fields - -{ref}`fields` represents the "space", aka. the radiance field of the NeRF. Here, we define the field as the part of the network that takes in point samples and any other conditioning, and outputs any of the `FieldHeadNames` (`nerf/field_modules/field_heads.py`). - -All fields are composed of modules continained within the `fields/` directory (e.g. `fields/modules/` or `fields/density_fields/`). We can think of field modules as modules that actually define or interact with the field. - -To build a NeRF field, we therefore follow these steps: - -1. define any relevant field modules -2. create a new class that extends the {ref}`fields` base class and use these field modules to compose the full field -3. implement the abstract functions `get_density()` and `get_outputs()`. - -```python -class NeRFField(Field): - """NeRF Field""" - - def __init__(self, ...) -> None: - """Create boilerplate NeRF field.""" - - super().__init__() - - # In the init method, we are instantiating all relevant field modules - self.position_encoding = position_encoding - self.direction_encoding = direction_encoding - - self.mlp_base = MLP( - in_dim=self.position_encoding.get_out_dim(), - num_layers=base_mlp_num_layers, - layer_width=base_mlp_layer_width, - skip_connections=skip_connections, - ) - ... - - def get_density(self, point_samples: PointSamples): - """ - Computes and returns the densities. - """ - - def get_outputs( - self, point_samples: PointSamples, density_embedding: Optional[TensorType] = None - ) -> Dict[FieldHeadNames, TensorType]: - """ - Computes and returns the outputs. - """ -``` - -### Putting it together - -Now that we've gone over what modules and fields are, we are ready to create the graph! - -To implement the vanilla NeRF, we create a new class that inherits the abstract Graph class. Similar to composing a new field, we can pull from all pre-defined fields and modules and use them to compose a new Graph. -We will then need to define the following abstract methods as seen in the skeleton code below. See also `graphs/vanilla_nerf.py` for the full implementation. - -```python -class NeRFGraph(Graph): - """Vanilla NeRF graph""" - - def __init__(self, intrinsics=None, camera_to_world=None, **kwargs) -> None: - super().__init__(intrinsics=intrinsics, camera_to_world=camera_to_world, **kwargs) - - def populate_fields(self): - """ - Set all field related modules here - """ - - def populate_misc_modules(self): - """ - Set all remaining modules here including: samplers, renderers, losses, and metrics - """ - - def get_param_groups(self) -> Dict[str, List[Parameter]]: - """ - Create a dictionary of parameters that are grouped according to different optimizers - """ - - def get_outputs(self, ray_bundle: RayBundle): - """ - Takes in a Ray Bundle and returns a dictionary of outputs. - """ - - def get_loss_dict(self, outputs, batch): - """ - Computes and returns the losses. - """ - - def log_test_image_outputs(self, image_idx, step, batch, outputs): - """ - Writes the test image outputs. - """ -``` - -## Creating the config - -Now that you have the graph and dataset all set up, you'll need to create the config that you pass into our run train routine. Our config system is powered by [Hydra](https://hydra.cc/). All Hydra and machine related arguments are stored in `configs/default_setup.yaml`, as well as the defaults list. -To set up the graph config, create a new yaml under `configs/`. - -```yaml -# configs/vanilla_nerf.yaml - -defaults: - - graph_default # inherit the basic yaml heirarchy - - _self_ - -logging: - # - -trainer: - # - -experiment_name: blender_lego -method_name: vanilla_nerf - -data: - # - -graph: - _target_: nerfactory.graph.vanilla_nerf.NeRFGraph # set the target to the graph you defined - # - -optimizers: - # -``` - -## Training and testing - -Once you have the config properly set up, you can begin training! To begin the training process, just pass in the name of your new config to the training script: - -```bash -python scripts/run_train.py --config-name vanilla_nerf -``` - -And now you have a brand-new NeRF in training! For testing and visualizing, simply refer to steps 4-5 in the [quickstart guide](https://github.com/plenoptix/nerfactory#quickstart). - -To help you get started, we also provide additional training tools such as profiling, logging, and debugging. Please refer to our [features guide](../../tooling/index.rst). diff --git a/docs/tutorials/models/creating_pipelines.rst b/docs/tutorials/models/creating_pipelines.rst deleted file mode 100644 index 7d791c33f..000000000 --- a/docs/tutorials/models/creating_pipelines.rst +++ /dev/null @@ -1,10 +0,0 @@ -Creating custom pipelines -------------------------- - -# TODO: high level summary - -.. toctree:: - :maxdepth: 1 - - system_overview.md - creating_graphs.md \ No newline at end of file diff --git a/docs/tutorials/models/system_overview.md b/docs/tutorials/models/system_overview.md deleted file mode 100644 index 4096fb3df..000000000 --- a/docs/tutorials/models/system_overview.md +++ /dev/null @@ -1,4 +0,0 @@ -# System Overview - -TODO: something with a higher-level node-graph image for wiring together the models -TODO: maybe also description of what the pipeline is constructed of \ No newline at end of file diff --git a/docs/tutorials/pipelines/creating_models.ipynb b/docs/tutorials/pipelines/creating_models.ipynb new file mode 100644 index 000000000..f5d41116a --- /dev/null +++ b/docs/tutorials/pipelines/creating_models.ipynb @@ -0,0 +1,229 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Developing Custom Models\n", + "\n", + "In this quick tour, we will walk you through the core of training and building any NeRFs with nerfactory.\n", + "\n", + "The entry point for training starts at `scripts/run_train.py`, which spawns instances of our `Trainer()` class (in `engine/trainer.py`). The `Trainer()` is responsible for setting up the datasets and NeRF graph depending on the config specified. It will then run the usual train/val routine for a config-specified number of iterations. If you are planning on using our codebase to build a new NeRF method or to use an existing implementation, we've abstracted away the training routine in these two files and chances are you will not need to think of them again.\n", + "\n", + "### Graphs, Modules, and Fields\n", + "\n", + "If you are looking to implemnet a new NeRF method or extend an existing one, you may need to set up a new graph (in `graphs/`).\n", + "\n", + "All NeRF graph definitions can be found in `graphs/`. The graph encapsulates all the modules in the NeRF method- from the MLP's to the samplers to the losses and optimizers. All graphs are composed of modules and fields.\n", + "\n", + "### Modules\n", + "\n", + "We can decompose all graphs into respective modules. For instance, we have {ref}`optimizers` which contains modules related to the loss, {ref}`renderers` which contains all rendering modules, {`graph/modules/`}, which includes samplers and ray generators. Additionally, we have {ref}`field_modules`, which will be discussed below.\n", + "In general, modules can be thought of as individual component parts of the graph that can be swapped in/out. We provide most of the standard modules, but if you are creating a new NeRF graph, you may have to define new modules.\n", + "\n", + "### Fields\n", + "\n", + "{ref}`fields` represents the \"space\", aka. the radiance field of the NeRF. Here, we define the field as the part of the network that takes in point samples and any other conditioning, and outputs any of the `FieldHeadNames` (`nerf/field_modules/field_heads.py`).\n", + "\n", + "All fields are composed of modules continained within the `fields/` directory (e.g. `fields/modules/` or `fields/density_fields/`). We can think of field modules as modules that actually define or interact with the field.\n", + "\n", + "To build a NeRF field, we therefore follow these steps:\n", + "\n", + "1. define any relevant field modules\n", + "2. create a new class that extends the {ref}`fields` base class and use these field modules to compose the full field\n", + "3. implement the abstract functions `get_density()` and `get_outputs()`." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "ename": "SyntaxError", + "evalue": "invalid syntax (383175768.py, line 4)", + "output_type": "error", + "traceback": [ + "\u001b[0;36m Input \u001b[0;32mIn [1]\u001b[0;36m\u001b[0m\n\u001b[0;31m def __init__(self, ...) -> None:\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n" + ] + } + ], + "source": [ + "class NeRFField(Field):\n", + " \"\"\"NeRF Field\"\"\"\n", + "\n", + " def __init__(self, ...) -> None:\n", + " \"\"\"Create boilerplate NeRF field.\"\"\"\n", + "\n", + " super().__init__()\n", + "\n", + " # In the init method, we are instantiating all relevant field modules\n", + " self.position_encoding = position_encoding\n", + " self.direction_encoding = direction_encoding\n", + "\n", + " self.mlp_base = MLP(\n", + " in_dim=self.position_encoding.get_out_dim(),\n", + " num_layers=base_mlp_num_layers,\n", + " layer_width=base_mlp_layer_width,\n", + " skip_connections=skip_connections,\n", + " )\n", + " ...\n", + "\n", + " def get_density(self, point_samples: PointSamples):\n", + " \"\"\"\n", + " Computes and returns the densities.\n", + " \"\"\"\n", + "\n", + " def get_outputs(\n", + " self, point_samples: PointSamples, density_embedding: Optional[TensorType] = None\n", + " ) -> Dict[FieldHeadNames, TensorType]:\n", + " \"\"\"\n", + " Computes and returns the outputs.\n", + " \"\"\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Putting it together\n", + "\n", + "Now that we've gone over what modules and fields are, we are ready to create the graph!\n", + "\n", + "To implement the vanilla NeRF, we create a new class that inherits the abstract Graph class. Similar to composing a new field, we can pull from all pre-defined fields and modules and use them to compose a new Graph.\n", + "We will then need to define the following abstract methods as seen in the skeleton code below. See also `graphs/vanilla_nerf.py` for the full implementation." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'Graph' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m/shared/ethanweber/nerfactory/docs/tutorials/pipelines/creating_models.ipynb Cell 4\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[39mclass\u001b[39;00m \u001b[39mNeRFGraph\u001b[39;00m(Graph):\n\u001b[1;32m 2\u001b[0m \u001b[39m\"\"\"Vanilla NeRF graph\"\"\"\u001b[39;00m\n\u001b[1;32m 4\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__init__\u001b[39m(\u001b[39mself\u001b[39m, intrinsics\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, camera_to_world\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mNone\u001b[39;00m:\n", + "\u001b[0;31mNameError\u001b[0m: name 'Graph' is not defined" + ] + } + ], + "source": [ + "class NeRFGraph(Graph):\n", + " \"\"\"Vanilla NeRF graph\"\"\"\n", + "\n", + " def __init__(self, intrinsics=None, camera_to_world=None, **kwargs) -> None:\n", + " super().__init__(intrinsics=intrinsics, camera_to_world=camera_to_world, **kwargs)\n", + "\n", + " def populate_fields(self):\n", + " \"\"\"\n", + " Set all field related modules here\n", + " \"\"\"\n", + "\n", + " def populate_misc_modules(self):\n", + " \"\"\"\n", + " Set all remaining modules here including: samplers, renderers, losses, and metrics\n", + " \"\"\"\n", + "\n", + " def get_param_groups(self) -> Dict[str, List[Parameter]]:\n", + " \"\"\"\n", + " Create a dictionary of parameters that are grouped according to different optimizers\n", + " \"\"\"\n", + "\n", + " def get_outputs(self, ray_bundle: RayBundle):\n", + " \"\"\"\n", + " Takes in a Ray Bundle and returns a dictionary of outputs.\n", + " \"\"\"\n", + "\n", + " def get_loss_dict(self, outputs, batch):\n", + " \"\"\"\n", + " Computes and returns the losses.\n", + " \"\"\"\n", + "\n", + " def log_test_image_outputs(self, image_idx, step, batch, outputs):\n", + " \"\"\"\n", + " Writes the test image outputs.\n", + " \"\"\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Creating the config\n", + "\n", + "Now that you have the graph and dataset all set up, you'll need to create the config that you pass into our run train routine. Our config system is powered by [Hydra](https://hydra.cc/). All Hydra and machine related arguments are stored in `configs/default_setup.yaml`, as well as the defaults list.\n", + "To set up the graph config, create a new yaml under `configs/`.\n", + "\n", + "```yaml\n", + "# configs/vanilla_nerf.yaml\n", + "\n", + "defaults:\n", + " - graph_default # inherit the basic yaml heirarchy\n", + " - _self_\n", + "\n", + "logging:\n", + " # \n", + "\n", + "trainer:\n", + " # \n", + "\n", + "experiment_name: blender_lego\n", + "method_name: vanilla_nerf\n", + "\n", + "data:\n", + " # \n", + "\n", + "graph:\n", + " _target_: nerfactory.graph.vanilla_nerf.NeRFGraph # set the target to the graph you defined\n", + " # \n", + "\n", + "optimizers:\n", + " # \n", + "```\n", + "\n", + "### Training and testing\n", + "\n", + "Once you have the config properly set up, you can begin training! To begin the training process, just pass in the name of your new config to the training script:\n", + "\n", + "```bash\n", + "python scripts/run_train.py --config-name vanilla_nerf\n", + "```\n", + "\n", + "And now you have a brand-new NeRF in training! For testing and visualizing, simply refer to steps 4-5 in the [quickstart guide](https://github.com/plenoptix/nerfactory#quickstart).\n", + "\n", + "To help you get started, we also provide additional training tools such as profiling, logging, and debugging. Please refer to our [features guide](../../tooling/index.rst)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8.13 ('nerfactory')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.13" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "e691f033e0f2f1b9c0a11e7e81375a1e27aec87a71fd1b1eaa545f7a5e61b3f6" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/tutorials/pipelines/creating_pipelines.ipynb b/docs/tutorials/pipelines/creating_pipelines.ipynb new file mode 100644 index 000000000..972bb54b9 --- /dev/null +++ b/docs/tutorials/pipelines/creating_pipelines.ipynb @@ -0,0 +1,276 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Creating Custom Pipelines\n", + "\n", + "Here we explain how to create custom pipelines in nerfactory. Pipelines are composed of two components, namely a Dataloader and a Model. We'll show how to make an incremental dataloader with multiple scenes, a model with scene conditioning, and importance sampling at pixels with high loss.\n", + "\n", + "The extensible features that we show in this tutorial are the following:\n", + "- a Dataloader that incrementally adds cameras\n", + "- a Dataloader that uses multiple scenes\n", + "- a Model that is conditioned on the scene\n", + "- a Dataloader PixelSampler that samples more densely in regions with high loss\n", + "\n", + "Our goal is to enable Nerfactory users to create an entire custom Pipeline by only editing one file.\n", + "\n", + "```text\n", + "#TODO(ethan): show a figure depicting the different modules that go into a pipeline\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "tags": [ + "remove-cell", + "hide-input" + ] + }, + "outputs": [], + "source": [ + "# COLLAPSED\n", + "%load_ext autoreload\n", + "%autoreload 2\n", + "\n", + "from __future__ import annotations\n", + "\n", + "from typing import Dict, Optional, Tuple, List\n", + "import torch\n", + "\n", + "from nerfactory.utils.config import DataloaderConfig, ModelConfig, PipelineConfig, ViewerConfig\n", + "\n", + "from nerfactory.dataloaders.base import Dataloader\n", + "from nerfactory.dataloaders.structs import DatasetInputs\n", + "from nerfactory.models.base import Model\n", + "from nerfactory.pipelines.base import Pipeline\n", + "\n", + "from nerfactory.models.instant_ngp import NGPModel\n", + "from nerfactory.cameras.rays import RayBundle" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### RayBundle class\n", + "\n", + "In the RayBundle class, we allow uses to add non-default attributes for conditioning.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class CustomDataloaderConfig(DataloaderConfig):\n", + " _target = CustomDataloader\n", + " scene_names: List[str] = [\"chair\", \"drums\", \"fern\", \"lego\"]\n", + " steps_per_add_cameras: int = 100 # every X iterations, add more cameras\n", + " num_cameras_per_add: int = 10 # number of cameras to add\n", + "\n", + "\n", + "class CustomDataloader(Dataloader):\n", + " \"\"\"A custom dataloader that incrementally adds cameras and has multiple scenes.\"\"\"\n", + "\n", + " config: CustomDataloaderConfig\n", + "\n", + " def populate_train_modules(self):\n", + " \"\"\"Populate the train dataloader modules.\"\"\"\n", + " self.scene_name_to_image_sampler = {}\n", + " self.scene_name_to_collider = {} # TODO(ethan): move colliders to the dataloader\n", + " for scene_name in self.config.scene_names:\n", + " dataset_inputs: DatasetInputs = None # TODO\n", + " image_dataset = ImageDataset(dataset_inputs)\n", + " image_sampler = CacheImageSampler(image_dataset=image_dataset) # a torch dataloader\n", + " self.scene_name_to_image_sampler[scene_name] = image_sampler\n", + " self.train_pixel_sampler = PixelSampler\n", + " self.train_ray_generator = RayGenerator # nn.Module\n", + "\n", + " def populate_eval_modules(self):\n", + " \"\"\"Populate the eval dataloader modules.\"\"\"\n", + " # we'll mostly rely on the train dataloader modules\n", + " self.eval_pixel_sampler = EvalPixelSampler # this will sample entire images\n", + "\n", + " def sample_images(self) -> List[Image]:\n", + " images = []\n", + " for scene_name in self.scene_name_to_image_sampler.keys():\n", + " image_sampler = self.scene_name_to_image_sampler[scene_name]\n", + " x = image_sampler.forward() # note that empty forward calls next()\n", + " images.append(x)\n", + " return images\n", + "\n", + " def next_train(self, step: int) -> Tuple[RayBundle, Dict]:\n", + " \"\"\"Get the next batch of training data by stringing together the train modules.\"\"\"\n", + " # grab some images from each scene\n", + " images = self.sample_images()\n", + " pixels = self.train_pixel_sampler.forward(images)\n", + " ray_bundle = self.train_ray_generator(pixels)\n", + " # shape should be (H, W, :)\n", + " ray_bundle.metadata[\"scene_indices\"] = None # TODO(ethan): populate this with a tensor of scene indices\n", + " dict_ = {}\n", + " return ray_bundle, dict_\n", + "\n", + " def next_eval(self, step: int) -> Tuple[RayBundle, Dict]:\n", + " \"\"\"Get the next batch of eval data by stringing together the eval modules.\"\"\"\n", + " # grab some images from each scene\n", + " images = self.sample_images()\n", + " pixels = self.train_pixel_sampler.forward(images)\n", + " ray_bundle = self.eval_ray_generator(pixels) # notice this is eval and not train\n", + " # shape should be (H, W, :)\n", + " ray_bundle.metadata[\"scene_indices\"] = None # TODO(ethan): populate this with a tensor of scene indices\n", + " dict_ = {}\n", + " return ray_bundle, dict_\n", + "\n", + "\n", + "class CustomNGPModelConfig(ModelConfig):\n", + " _target = CustomNGPModel\n", + " coarse_field: str = \"temp\"\n", + " fine_field: str = \"temp2\"\n", + "\n", + "\n", + "class CustomNGPModel(Model):\n", + " \"\"\"An instant ngp model modified slightly to output semantics.\"\"\"\n", + "\n", + " config: SceneConditionNGPModelConfig\n", + "\n", + " def populate_modules(self):\n", + " self.ngp_model = NGPModel(coarse_field=self.config.coarse_field, fine_field=self.config.fine_field)\n", + "\n", + " def get_outputs(self, ray_bundle: RayBundle) -> Dict[str, torch.Tensor]:\n", + " # TODO(ethan): pass in batch from the forward function\n", + " # TODO(ethan): rename batch to something else\n", + " outputs = self.ngp_model.forward(ray_bundle)\n", + " outputs[\"semantics\"] = torch.rand_like(outputs[\"rgb\"])\n", + " return outputs\n", + "\n", + " def get_loss_dict(self):\n", + " return {}\n", + "\n", + "\n", + "class CustomPipeline(Pipeline):\n", + " \"\"\"The Instant NGP pipeline.\"\"\"\n", + "\n", + " config: PipelineConfig\n", + "\n", + " def train_step(self, step: int):\n", + " ray_bundle, batch = self.dataloader.next_train(step=step)\n", + " # TODO: maybe run a CNN on the data before passing into model\n", + " model_outputs, loss_dict, metrics_dict = self.model(ray_bundle, batch)\n", + " # TODO: update pixel sampler state with loss map to show the flexibilty of our Pipeline\n", + " self.dataloader.pixel_sampler.update_loss_map()\n", + " return model_outputs, loss_dict, metrics_dict\n", + "\n", + " def eval_step(self):\n", + " ray_bundle, batch = self.dataloader.next_eval(step=step)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dataloader = Dataloader()\n", + "model = SceneConditionNGPModel()\n", + "pipeline = CustomPipeline.from_dataloader_and_model(dataloader=dataloader, model=model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Creating pipelines from a config\n", + "\n", + "Now we show how to create a pipeline from a config, which has the following form:\n", + "\n", + "```python\n", + "@dataclass\n", + "class PipelineConfig:\n", + " \"\"\"Configuration for pipeline instantiation\"\"\"\n", + "\n", + " _target: ClassVar[Type] = Pipeline\n", + " dataloader: DataloaderConfig = MISSING\n", + " model: ModelConfig = MISSING\n", + "```\n", + "\n", + "See `nerfactory/utils/config.py` for more details. In this example, we will simply load from an existing configuration from `configs/graph_instant_ngp.yaml`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pprint\n", + "import hydra\n", + "\n", + "hydra.core.global_hydra.GlobalHydra.instance().clear()\n", + "from hydra import compose, initialize\n", + "\n", + "initialize(version_base=\"1.2\", config_path=\"../configs/\")\n", + "config_name = \"graph_instant_ngp.yaml\"\n", + "config = compose(config_name)\n", + "pipeline_config = config.pipeline\n", + "pprint.pprint(pipeline_config)\n", + "print(\"----------------------------------------------------\")\n", + "\n", + "# from nerfactory.pipelines.base import setup_pipeline\n", + "# pipeline = setup_pipeline(pipeline_config, device=\"cuda\")\n", + "pipeline = pipeline_config.setup()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pipeline.get_train_loss_dict()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8.13 ('nerfactory')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.13" + }, + "vscode": { + "interpreter": { + "hash": "e691f033e0f2f1b9c0a11e7e81375a1e27aec87a71fd1b1eaa545f7a5e61b3f6" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/tutorials/pipelines/index.rst b/docs/tutorials/pipelines/index.rst new file mode 100644 index 000000000..de061bb58 --- /dev/null +++ b/docs/tutorials/pipelines/index.rst @@ -0,0 +1,10 @@ +Creating Custom Pipelines +------------------------- + +Here we show you how to create a custom pipeline. + +.. toctree:: + :maxdepth: 1 + + creating_pipelines.ipynb + creating_models.ipynb \ No newline at end of file diff --git a/docs/tutorials/viewer/using_viewer.rst b/docs/tutorials/viewer/index.rst similarity index 100% rename from docs/tutorials/viewer/using_viewer.rst rename to docs/tutorials/viewer/index.rst diff --git a/notebooks/creating_pipelines.ipynb b/notebooks/creating_pipelines.ipynb deleted file mode 100644 index 6bc021f23..000000000 --- a/notebooks/creating_pipelines.ipynb +++ /dev/null @@ -1,240 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Creating pipelines\n", - "\n", - "Here we explain how to create custom pipelines in nerfactory. Pipelines are composed of two components, namely a Dataloader and a Model." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The autoreload extension is already loaded. To reload it, use:\n", - " %reload_ext autoreload\n" - ] - } - ], - "source": [ - "# HIDDEN\n", - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "from typing import Dict\n", - "import torch\n", - "\n", - "from nerfactory.dataloaders.base import Dataloader\n", - "from nerfactory.models.base import Model\n", - "from nerfactory.pipelines.base import Pipeline\n", - "\n", - "from nerfactory.models.instant_ngp import NGPModel\n", - "from nerfactory.cameras.rays import RayBundle\n", - "\n", - "\n", - "class SemanticNGPModel(Model):\n", - " \"\"\"An instant ngp model modified slightly to output semantics.\"\"\"\n", - "\n", - " def __init__(self):\n", - " self.ngp_model = NGPModel()\n", - "\n", - " def get_outputs(self, ray_bundle: RayBundle) -> Dict[str, torch.Tensor]:\n", - " outputs = self.ngp_model.forward(ray_bundle)\n", - " outputs[\"semantics\"] = torch.rand_like(outputs[\"rgb\"])\n", - " return outputs\n", - "\n", - "\n", - "class CustomPipeline(Pipeline):\n", - " \"\"\"The Instant NGP pipeline.\"\"\"\n", - "\n", - " dataloader: Dataloader\n", - " model: Model\n", - "\n", - " def get_train_loss_dict(self):\n", - " rays, batch = self.dataloader_train_iter.next()\n", - " # TODO: do something here\n", - " accumulated_color, _, _, mask = self.model(rays, batch)\n", - " masked_batch = get_masked_dict(batch, mask)\n", - " loss_dict = self.model.get_loss_dict(accumulated_color, masked_batch, mask)\n", - " return loss_dict\n", - "\n", - " def get_eval_loss_dict(self):\n", - " rays, batch = self.dataloader_eval_iter.next()\n", - " # TODO: do something here\n", - " accumulated_color, _, _, mask = self.model(rays, batch)\n", - " masked_batch = get_masked_dict(batch, mask)\n", - " loss_dict = self.model.get_loss_dict(accumulated_color, masked_batch, mask)\n", - " return loss_dict\n", - "\n", - " def test_image_outputs(self) -> None:\n", - " \"\"\"Log the test image outputs\"\"\"\n", - " # camera = Camera(self.dataloader.eval_datasetinputs.camera_to_world[0])\n", - " pass" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "ename": "NotImplementedError", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNotImplementedError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb Cell 4\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m dataloader \u001b[39m=\u001b[39m Dataloader(use_train\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m, use_eval\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n\u001b[1;32m 2\u001b[0m model \u001b[39m=\u001b[39m SemanticNGPModel()\n\u001b[1;32m 3\u001b[0m pipeline \u001b[39m=\u001b[39m CustomPipeline(dataloader\u001b[39m=\u001b[39mdataloader, model\u001b[39m=\u001b[39mmodel)\n", - "File \u001b[0;32m/shared/ethanweber/nerfactory/nerfactory/dataloaders/base.py:95\u001b[0m, in \u001b[0;36mDataloader.__init__\u001b[0;34m(self, use_train, use_eval)\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[39massert\u001b[39;00m use_train \u001b[39mor\u001b[39;00m use_eval\n\u001b[1;32m 94\u001b[0m \u001b[39mif\u001b[39;00m use_train:\n\u001b[0;32m---> 95\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msetup_train()\n\u001b[1;32m 96\u001b[0m \u001b[39mif\u001b[39;00m use_eval:\n\u001b[1;32m 97\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39msetup_eval()\n", - "File \u001b[0;32m/shared/ethanweber/nerfactory/nerfactory/dataloaders/base.py:116\u001b[0m, in \u001b[0;36mDataloader.setup_train\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[39m@abstractmethod\u001b[39m\n\u001b[1;32m 114\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39msetup_train\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 115\u001b[0m \u001b[39m\"\"\"Sets up the dataloader for training\"\"\"\u001b[39;00m\n\u001b[0;32m--> 116\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mNotImplementedError\u001b[39;00m\n", - "\u001b[0;31mNotImplementedError\u001b[0m: " - ] - } - ], - "source": [ - "dataloader = Dataloader(use_train=True, use_eval=True)\n", - "model = SemanticNGPModel()\n", - "pipeline = CustomPipeline(dataloader=dataloader, model=model)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Creating pipelines from a config\n", - "\n", - "Now we show how to create a pipeline from a config, which has the following form:\n", - "\n", - "```python\n", - "@dataclass\n", - "class PipelineConfig:\n", - " \"\"\"Configuration for pipeline instantiation\"\"\"\n", - "\n", - " _target_: str = MISSING\n", - " dataloader_config: DataloaderConfig = MISSING\n", - " graph_config: GraphConfig = MISSING\n", - "```\n", - "\n", - "See [nerfactory/utils/config.py](nerfactory/utils/config.py) for more details. In this example, we will simply load from an existing configuration from [configs/graph_instant_ngp.yaml](configs/graph_instant_ngp.yaml)." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'_target_': 'nerfactory.pipelines.base.Pipeline', 'dataloader': {'_target_': 'nerfactory.dataloaders.base.VanillaDataloader', 'train_dataset': {'_target_': 'nerfactory.dataloaders.datasets.Blender', 'data_directory': 'data/blender/lego', 'alpha_color': 'white', 'downscale_factor': 1}, 'train_num_rays_per_batch': 8192, 'eval_dataset': {'_target_': 'nerfactory.dataloaders.datasets.Blender', 'data_directory': 'data/blender/lego', 'alpha_color': 'white', 'downscale_factor': 1}, 'eval_num_rays_per_chunk': 4096, 'eval_num_rays_per_chunk': 8192}, 'model': {'_target_': 'nerfactory.models.instant_ngp.NGPModel', 'enable_collider': False, 'collider_config': {'_target_': 'nerfactory.models.modules.scene_colliders.NearFarCollider', 'near_plane': 2.0, 'far_plane': 6.0}, 'num_coarse_samples': 64, 'num_importance_samples': 128, 'loss_coefficients': {'rgb_loss_coarse': 1.0, 'rgb_loss_fine': 1.0, 'rgb_loss': 1.0}, 'enable_density_field': True, 'density_field_config': {'_target_': 'nerfactory.fields.density_fields.density_grid.DensityGrid', 'center': 0.0, 'base_scale': 3.0, 'num_cascades': 1, 'resolution': 128, 'update_every_num_iters': 16}, 'field_implementation': 'tcnn'}}\n", - "----------------------------------------------------\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/shared/ethanweber/miniconda3/envs/nerfactory/lib/python3.8/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and will be removed in 0.15, please use 'weights' instead.\n", - " warnings.warn(\n", - "/shared/ethanweber/miniconda3/envs/nerfactory/lib/python3.8/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and will be removed in 0.15. The current behavior is equivalent to passing `weights=AlexNet_Weights.IMAGENET1K_V1`. You can also use `weights=AlexNet_Weights.DEFAULT` to get the most up-to-date weights.\n", - " warnings.warn(msg)\n" - ] - } - ], - "source": [ - "import pprint\n", - "import hydra\n", - "\n", - "hydra.core.global_hydra.GlobalHydra.instance().clear()\n", - "from hydra import compose, initialize\n", - "\n", - "initialize(version_base=\"1.2\", config_path=\"../configs/\")\n", - "config_name = \"graph_instant_ngp.yaml\"\n", - "config = compose(config_name)\n", - "pipeline_config = config.pipeline\n", - "pprint.pprint(pipeline_config)\n", - "print(\"----------------------------------------------------\")\n", - "\n", - "from nerfactory.pipelines.base import setup_pipeline\n", - "\n", - "pipeline = setup_pipeline(pipeline_config, device=\"cuda\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "ename": "AttributeError", - "evalue": "'IterableWrapper' object has no attribute 'next'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb Cell 8\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m pipeline\u001b[39m.\u001b[39;49mget_train_loss_dict()\n", - "File \u001b[0;32m/shared/ethanweber/nerfactory/nerfactory/pipelines/base.py:81\u001b[0m, in \u001b[0;36mPipeline.get_train_loss_dict\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 76\u001b[0m \u001b[39m@abstractmethod\u001b[39m\n\u001b[1;32m 77\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mget_train_loss_dict\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 78\u001b[0m \u001b[39m\"\"\"This function gets your training loss dict. This will be responsible for\u001b[39;00m\n\u001b[1;32m 79\u001b[0m \u001b[39m getting the next batch of data from the dataloader and interfacing with the\u001b[39;00m\n\u001b[1;32m 80\u001b[0m \u001b[39m Model class, feeding the data to the model's forward function.\"\"\"\u001b[39;00m\n\u001b[0;32m---> 81\u001b[0m rays, batch \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdataloader_train_iter\u001b[39m.\u001b[39;49mnext()\n\u001b[1;32m 82\u001b[0m accumulated_color, _, _, mask \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel(rays, batch)\n\u001b[1;32m 83\u001b[0m masked_batch \u001b[39m=\u001b[39m get_masked_dict(batch, mask)\n", - "\u001b[0;31mAttributeError\u001b[0m: 'IterableWrapper' object has no attribute 'next'" - ] - } - ], - "source": [ - "pipeline.get_train_loss_dict()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.8.13 ('nerfactory')", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.13" - }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "e691f033e0f2f1b9c0a11e7e81375a1e27aec87a71fd1b1eaa545f7a5e61b3f6" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} \ No newline at end of file