Files
nerfstudio/notebooks/creating_pipelines.ipynb
T
Ethan WeberandJake Austin 0b042be956 Pipeline v2 (#258)
* Model class done, pipeline class still needs more visualization stuffs

* Renaming the BaseDataContainer to DataloaderOutputs and adding documentation

* Pushing more documentation, removing some trivial functions

* Updating dict typing

* Removing config dicts for collider

* Passing tests

* Pushing shell new dataloader

* Saving work, need to fix NGP model's get_loss_dict and Pipeline

* V1 of the ngp pipeline should be done. Need train script and debugging

* Updating with latest instant-ngp code

* Syntax updates from latest ngp graph code

* Linter, removing unneeded code in new NGP graph code

* Style

* Should be good now. Linter is happy and the code is verbatum current instant-ngp

* Everything done except novel view testing code

* Fixing some nitpicks

* Debugging dataloader

* merged master

* WOP. integrating pipeline with configs and notebook demo

* removing model folder

* a lot of name changing from Graph to Model

* in progress... changing trainer to use pipeline... broken right now

* pipeline code runs

* switched to val dataset

* makes run_eval.py work

* updated configs for pipeline change

* fixing test_run_train script

* fixing black and linting

* fix docs

* small updates to trainer

Co-authored-by: Jake Austin <jake-austin@berkeley.edu>
2022-08-10 14:59:45 -07:00

240 lines
12 KiB
Plaintext

{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Creating pipelines\n",
"\n",
"Here we explain how to create custom pipelines in nerfactory. Pipelines are composed of two components, namely a Dataloader and a Model."
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"The autoreload extension is already loaded. To reload it, use:\n",
" %reload_ext autoreload\n"
]
}
],
"source": [
"# HIDDEN\n",
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"from typing import Dict\n",
"import torch\n",
"\n",
"from nerfactory.dataloaders.base import Dataloader\n",
"from nerfactory.models.base import Model\n",
"from nerfactory.pipelines.base import Pipeline\n",
"\n",
"from nerfactory.models.instant_ngp import NGPModel\n",
"from nerfactory.cameras.rays import RayBundle\n",
"\n",
"\n",
"class SemanticNGPModel(Model):\n",
" \"\"\"An instant ngp model modified slightly to output semantics.\"\"\"\n",
"\n",
" def __init__(self):\n",
" self.ngp_model = NGPModel()\n",
"\n",
" def get_outputs(self, ray_bundle: RayBundle) -> Dict[str, torch.Tensor]:\n",
" outputs = self.ngp_model.forward(ray_bundle)\n",
" outputs[\"semantics\"] = torch.rand_like(outputs[\"rgb\"])\n",
" return outputs\n",
"\n",
"\n",
"class CustomPipeline(Pipeline):\n",
" \"\"\"The Instant NGP pipeline.\"\"\"\n",
"\n",
" dataloader: Dataloader\n",
" model: Model\n",
"\n",
" def get_train_loss_dict(self):\n",
" rays, batch = self.dataloader_train_iter.next()\n",
" # TODO: do something here\n",
" accumulated_color, _, _, mask = self.model(rays, batch)\n",
" masked_batch = get_masked_dict(batch, mask)\n",
" loss_dict = self.model.get_loss_dict(accumulated_color, masked_batch, mask)\n",
" return loss_dict\n",
"\n",
" def get_eval_loss_dict(self):\n",
" rays, batch = self.dataloader_eval_iter.next()\n",
" # TODO: do something here\n",
" accumulated_color, _, _, mask = self.model(rays, batch)\n",
" masked_batch = get_masked_dict(batch, mask)\n",
" loss_dict = self.model.get_loss_dict(accumulated_color, masked_batch, mask)\n",
" return loss_dict\n",
"\n",
" def test_image_outputs(self) -> None:\n",
" \"\"\"Log the test image outputs\"\"\"\n",
" # camera = Camera(self.dataloader.eval_datasetinputs.camera_to_world[0])\n",
" pass"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"ename": "NotImplementedError",
"evalue": "",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mNotImplementedError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb Cell 4\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2Bbirman/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb#W3sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m dataloader \u001b[39m=\u001b[39m Dataloader(use_train\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m, use_eval\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bbirman/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb#W3sdnNjb2RlLXJlbW90ZQ%3D%3D?line=1'>2</a>\u001b[0m model \u001b[39m=\u001b[39m SemanticNGPModel()\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bbirman/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb#W3sdnNjb2RlLXJlbW90ZQ%3D%3D?line=2'>3</a>\u001b[0m pipeline \u001b[39m=\u001b[39m CustomPipeline(dataloader\u001b[39m=\u001b[39mdataloader, model\u001b[39m=\u001b[39mmodel)\n",
"File \u001b[0;32m/shared/ethanweber/nerfactory/nerfactory/dataloaders/base.py:95\u001b[0m, in \u001b[0;36mDataloader.__init__\u001b[0;34m(self, use_train, use_eval)\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[39massert\u001b[39;00m use_train \u001b[39mor\u001b[39;00m use_eval\n\u001b[1;32m 94\u001b[0m \u001b[39mif\u001b[39;00m use_train:\n\u001b[0;32m---> 95\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msetup_train()\n\u001b[1;32m 96\u001b[0m \u001b[39mif\u001b[39;00m use_eval:\n\u001b[1;32m 97\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39msetup_eval()\n",
"File \u001b[0;32m/shared/ethanweber/nerfactory/nerfactory/dataloaders/base.py:116\u001b[0m, in \u001b[0;36mDataloader.setup_train\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[39m@abstractmethod\u001b[39m\n\u001b[1;32m 114\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39msetup_train\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 115\u001b[0m \u001b[39m\"\"\"Sets up the dataloader for training\"\"\"\u001b[39;00m\n\u001b[0;32m--> 116\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mNotImplementedError\u001b[39;00m\n",
"\u001b[0;31mNotImplementedError\u001b[0m: "
]
}
],
"source": [
"dataloader = Dataloader(use_train=True, use_eval=True)\n",
"model = SemanticNGPModel()\n",
"pipeline = CustomPipeline(dataloader=dataloader, model=model)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Creating pipelines from a config\n",
"\n",
"Now we show how to create a pipeline from a config, which has the following form:\n",
"\n",
"```python\n",
"@dataclass\n",
"class PipelineConfig:\n",
" \"\"\"Configuration for pipeline instantiation\"\"\"\n",
"\n",
" _target_: str = MISSING\n",
" dataloader_config: DataloaderConfig = MISSING\n",
" graph_config: GraphConfig = MISSING\n",
"```\n",
"\n",
"See [nerfactory/utils/config.py](nerfactory/utils/config.py) for more details. In this example, we will simply load from an existing configuration from [configs/graph_instant_ngp.yaml](configs/graph_instant_ngp.yaml)."
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'_target_': 'nerfactory.pipelines.base.Pipeline', 'dataloader': {'_target_': 'nerfactory.dataloaders.base.VanillaDataloader', 'train_dataset': {'_target_': 'nerfactory.dataloaders.datasets.Blender', 'data_directory': 'data/blender/lego', 'alpha_color': 'white', 'downscale_factor': 1}, 'train_num_rays_per_batch': 8192, 'eval_dataset': {'_target_': 'nerfactory.dataloaders.datasets.Blender', 'data_directory': 'data/blender/lego', 'alpha_color': 'white', 'downscale_factor': 1}, 'eval_num_rays_per_chunk': 4096, 'eval_num_rays_per_chunk': 8192}, 'model': {'_target_': 'nerfactory.models.instant_ngp.NGPModel', 'enable_collider': False, 'collider_config': {'_target_': 'nerfactory.models.modules.scene_colliders.NearFarCollider', 'near_plane': 2.0, 'far_plane': 6.0}, 'num_coarse_samples': 64, 'num_importance_samples': 128, 'loss_coefficients': {'rgb_loss_coarse': 1.0, 'rgb_loss_fine': 1.0, 'rgb_loss': 1.0}, 'enable_density_field': True, 'density_field_config': {'_target_': 'nerfactory.fields.density_fields.density_grid.DensityGrid', 'center': 0.0, 'base_scale': 3.0, 'num_cascades': 1, 'resolution': 128, 'update_every_num_iters': 16}, 'field_implementation': 'tcnn'}}\n",
"----------------------------------------------------\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/shared/ethanweber/miniconda3/envs/nerfactory/lib/python3.8/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and will be removed in 0.15, please use 'weights' instead.\n",
" warnings.warn(\n",
"/shared/ethanweber/miniconda3/envs/nerfactory/lib/python3.8/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and will be removed in 0.15. The current behavior is equivalent to passing `weights=AlexNet_Weights.IMAGENET1K_V1`. You can also use `weights=AlexNet_Weights.DEFAULT` to get the most up-to-date weights.\n",
" warnings.warn(msg)\n"
]
}
],
"source": [
"import pprint\n",
"import hydra\n",
"\n",
"hydra.core.global_hydra.GlobalHydra.instance().clear()\n",
"from hydra import compose, initialize\n",
"\n",
"initialize(version_base=\"1.2\", config_path=\"../configs/\")\n",
"config_name = \"graph_instant_ngp.yaml\"\n",
"config = compose(config_name)\n",
"pipeline_config = config.pipeline\n",
"pprint.pprint(pipeline_config)\n",
"print(\"----------------------------------------------------\")\n",
"\n",
"from nerfactory.pipelines.base import setup_pipeline\n",
"\n",
"pipeline = setup_pipeline(pipeline_config, device=\"cuda\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": []
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"ename": "AttributeError",
"evalue": "'IterableWrapper' object has no attribute 'next'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb Cell 8\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2Bbirman/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb#X13sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m pipeline\u001b[39m.\u001b[39;49mget_train_loss_dict()\n",
"File \u001b[0;32m/shared/ethanweber/nerfactory/nerfactory/pipelines/base.py:81\u001b[0m, in \u001b[0;36mPipeline.get_train_loss_dict\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 76\u001b[0m \u001b[39m@abstractmethod\u001b[39m\n\u001b[1;32m 77\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mget_train_loss_dict\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 78\u001b[0m \u001b[39m\"\"\"This function gets your training loss dict. This will be responsible for\u001b[39;00m\n\u001b[1;32m 79\u001b[0m \u001b[39m getting the next batch of data from the dataloader and interfacing with the\u001b[39;00m\n\u001b[1;32m 80\u001b[0m \u001b[39m Model class, feeding the data to the model's forward function.\"\"\"\u001b[39;00m\n\u001b[0;32m---> 81\u001b[0m rays, batch \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdataloader_train_iter\u001b[39m.\u001b[39;49mnext()\n\u001b[1;32m 82\u001b[0m accumulated_color, _, _, mask \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel(rays, batch)\n\u001b[1;32m 83\u001b[0m masked_batch \u001b[39m=\u001b[39m get_masked_dict(batch, mask)\n",
"\u001b[0;31mAttributeError\u001b[0m: 'IterableWrapper' object has no attribute 'next'"
]
}
],
"source": [
"pipeline.get_train_loss_dict()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.8.13 ('nerfactory')",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.13"
},
"orig_nbformat": 4,
"vscode": {
"interpreter": {
"hash": "e691f033e0f2f1b9c0a11e7e81375a1e27aec87a71fd1b1eaa545f7a5e61b3f6"
}
}
},
"nbformat": 4,
"nbformat_minor": 2
}