{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Creating pipelines\n", "\n", "Here we explain how to create custom pipelines in nerfactory. Pipelines are composed of two components, namely a Dataloader and a Model." ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "The autoreload extension is already loaded. To reload it, use:\n", " %reload_ext autoreload\n" ] } ], "source": [ "# HIDDEN\n", "%load_ext autoreload\n", "%autoreload 2" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "from typing import Dict\n", "import torch\n", "\n", "from nerfactory.dataloaders.base import Dataloader\n", "from nerfactory.models.base import Model\n", "from nerfactory.pipelines.base import Pipeline\n", "\n", "from nerfactory.models.instant_ngp import NGPModel\n", "from nerfactory.cameras.rays import RayBundle\n", "\n", "\n", "class SemanticNGPModel(Model):\n", " \"\"\"An instant ngp model modified slightly to output semantics.\"\"\"\n", "\n", " def __init__(self):\n", " self.ngp_model = NGPModel()\n", "\n", " def get_outputs(self, ray_bundle: RayBundle) -> Dict[str, torch.Tensor]:\n", " outputs = self.ngp_model.forward(ray_bundle)\n", " outputs[\"semantics\"] = torch.rand_like(outputs[\"rgb\"])\n", " return outputs\n", "\n", "\n", "class CustomPipeline(Pipeline):\n", " \"\"\"The Instant NGP pipeline.\"\"\"\n", "\n", " dataloader: Dataloader\n", " model: Model\n", "\n", " def get_train_loss_dict(self):\n", " rays, batch = self.dataloader_train_iter.next()\n", " # TODO: do something here\n", " accumulated_color, _, _, mask = self.model(rays, batch)\n", " masked_batch = get_masked_dict(batch, mask)\n", " loss_dict = self.model.get_loss_dict(accumulated_color, masked_batch, mask)\n", " return loss_dict\n", "\n", " def get_eval_loss_dict(self):\n", " rays, batch = self.dataloader_eval_iter.next()\n", " # TODO: do something here\n", " accumulated_color, _, _, mask = self.model(rays, batch)\n", " masked_batch = get_masked_dict(batch, mask)\n", " loss_dict = self.model.get_loss_dict(accumulated_color, masked_batch, mask)\n", " return loss_dict\n", "\n", " def test_image_outputs(self) -> None:\n", " \"\"\"Log the test image outputs\"\"\"\n", " # camera = Camera(self.dataloader.eval_datasetinputs.camera_to_world[0])\n", " pass" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "ename": "NotImplementedError", "evalue": "", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNotImplementedError\u001b[0m Traceback (most recent call last)", "\u001b[1;32m/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb Cell 4\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m dataloader \u001b[39m=\u001b[39m Dataloader(use_train\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m, use_eval\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n\u001b[1;32m 2\u001b[0m model \u001b[39m=\u001b[39m SemanticNGPModel()\n\u001b[1;32m 3\u001b[0m pipeline \u001b[39m=\u001b[39m CustomPipeline(dataloader\u001b[39m=\u001b[39mdataloader, model\u001b[39m=\u001b[39mmodel)\n", "File \u001b[0;32m/shared/ethanweber/nerfactory/nerfactory/dataloaders/base.py:95\u001b[0m, in \u001b[0;36mDataloader.__init__\u001b[0;34m(self, use_train, use_eval)\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[39massert\u001b[39;00m use_train \u001b[39mor\u001b[39;00m use_eval\n\u001b[1;32m 94\u001b[0m \u001b[39mif\u001b[39;00m use_train:\n\u001b[0;32m---> 95\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msetup_train()\n\u001b[1;32m 96\u001b[0m \u001b[39mif\u001b[39;00m use_eval:\n\u001b[1;32m 97\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39msetup_eval()\n", "File \u001b[0;32m/shared/ethanweber/nerfactory/nerfactory/dataloaders/base.py:116\u001b[0m, in \u001b[0;36mDataloader.setup_train\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[39m@abstractmethod\u001b[39m\n\u001b[1;32m 114\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39msetup_train\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 115\u001b[0m \u001b[39m\"\"\"Sets up the dataloader for training\"\"\"\u001b[39;00m\n\u001b[0;32m--> 116\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mNotImplementedError\u001b[39;00m\n", "\u001b[0;31mNotImplementedError\u001b[0m: " ] } ], "source": [ "dataloader = Dataloader(use_train=True, use_eval=True)\n", "model = SemanticNGPModel()\n", "pipeline = CustomPipeline(dataloader=dataloader, model=model)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Creating pipelines from a config\n", "\n", "Now we show how to create a pipeline from a config, which has the following form:\n", "\n", "```python\n", "@dataclass\n", "class PipelineConfig:\n", " \"\"\"Configuration for pipeline instantiation\"\"\"\n", "\n", " _target_: str = MISSING\n", " dataloader_config: DataloaderConfig = MISSING\n", " graph_config: GraphConfig = MISSING\n", "```\n", "\n", "See [nerfactory/utils/config.py](nerfactory/utils/config.py) for more details. In this example, we will simply load from an existing configuration from [configs/graph_instant_ngp.yaml](configs/graph_instant_ngp.yaml)." ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'_target_': 'nerfactory.pipelines.base.Pipeline', 'dataloader': {'_target_': 'nerfactory.dataloaders.base.VanillaDataloader', 'train_dataset': {'_target_': 'nerfactory.dataloaders.datasets.Blender', 'data_directory': 'data/blender/lego', 'alpha_color': 'white', 'downscale_factor': 1}, 'train_num_rays_per_batch': 8192, 'eval_dataset': {'_target_': 'nerfactory.dataloaders.datasets.Blender', 'data_directory': 'data/blender/lego', 'alpha_color': 'white', 'downscale_factor': 1}, 'eval_num_rays_per_chunk': 4096, 'eval_num_rays_per_chunk': 8192}, 'model': {'_target_': 'nerfactory.models.instant_ngp.NGPModel', 'enable_collider': False, 'collider_config': {'_target_': 'nerfactory.models.modules.scene_colliders.NearFarCollider', 'near_plane': 2.0, 'far_plane': 6.0}, 'num_coarse_samples': 64, 'num_importance_samples': 128, 'loss_coefficients': {'rgb_loss_coarse': 1.0, 'rgb_loss_fine': 1.0, 'rgb_loss': 1.0}, 'enable_density_field': True, 'density_field_config': {'_target_': 'nerfactory.fields.density_fields.density_grid.DensityGrid', 'center': 0.0, 'base_scale': 3.0, 'num_cascades': 1, 'resolution': 128, 'update_every_num_iters': 16}, 'field_implementation': 'tcnn'}}\n", "----------------------------------------------------\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/shared/ethanweber/miniconda3/envs/nerfactory/lib/python3.8/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and will be removed in 0.15, please use 'weights' instead.\n", " warnings.warn(\n", "/shared/ethanweber/miniconda3/envs/nerfactory/lib/python3.8/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and will be removed in 0.15. The current behavior is equivalent to passing `weights=AlexNet_Weights.IMAGENET1K_V1`. You can also use `weights=AlexNet_Weights.DEFAULT` to get the most up-to-date weights.\n", " warnings.warn(msg)\n" ] } ], "source": [ "import pprint\n", "import hydra\n", "\n", "hydra.core.global_hydra.GlobalHydra.instance().clear()\n", "from hydra import compose, initialize\n", "\n", "initialize(version_base=\"1.2\", config_path=\"../configs/\")\n", "config_name = \"graph_instant_ngp.yaml\"\n", "config = compose(config_name)\n", "pipeline_config = config.pipeline\n", "pprint.pprint(pipeline_config)\n", "print(\"----------------------------------------------------\")\n", "\n", "from nerfactory.pipelines.base import setup_pipeline\n", "\n", "pipeline = setup_pipeline(pipeline_config, device=\"cuda\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "ename": "AttributeError", "evalue": "'IterableWrapper' object has no attribute 'next'", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", "\u001b[1;32m/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb Cell 8\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m pipeline\u001b[39m.\u001b[39;49mget_train_loss_dict()\n", "File \u001b[0;32m/shared/ethanweber/nerfactory/nerfactory/pipelines/base.py:81\u001b[0m, in \u001b[0;36mPipeline.get_train_loss_dict\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 76\u001b[0m \u001b[39m@abstractmethod\u001b[39m\n\u001b[1;32m 77\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mget_train_loss_dict\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 78\u001b[0m \u001b[39m\"\"\"This function gets your training loss dict. This will be responsible for\u001b[39;00m\n\u001b[1;32m 79\u001b[0m \u001b[39m getting the next batch of data from the dataloader and interfacing with the\u001b[39;00m\n\u001b[1;32m 80\u001b[0m \u001b[39m Model class, feeding the data to the model's forward function.\"\"\"\u001b[39;00m\n\u001b[0;32m---> 81\u001b[0m rays, batch \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdataloader_train_iter\u001b[39m.\u001b[39;49mnext()\n\u001b[1;32m 82\u001b[0m accumulated_color, _, _, mask \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel(rays, batch)\n\u001b[1;32m 83\u001b[0m masked_batch \u001b[39m=\u001b[39m get_masked_dict(batch, mask)\n", "\u001b[0;31mAttributeError\u001b[0m: 'IterableWrapper' object has no attribute 'next'" ] } ], "source": [ "pipeline.get_train_loss_dict()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3.8.13 ('nerfactory')", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.13" }, "orig_nbformat": 4, "vscode": { "interpreter": { "hash": "e691f033e0f2f1b9c0a11e7e81375a1e27aec87a71fd1b1eaa545f7a5e61b3f6" } } }, "nbformat": 4, "nbformat_minor": 2 }