mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-19 22:25:44 +08:00
* Model class done, pipeline class still needs more visualization stuffs * Renaming the BaseDataContainer to DataloaderOutputs and adding documentation * Pushing more documentation, removing some trivial functions * Updating dict typing * Removing config dicts for collider * Passing tests * Pushing shell new dataloader * Saving work, need to fix NGP model's get_loss_dict and Pipeline * V1 of the ngp pipeline should be done. Need train script and debugging * Updating with latest instant-ngp code * Syntax updates from latest ngp graph code * Linter, removing unneeded code in new NGP graph code * Style * Should be good now. Linter is happy and the code is verbatum current instant-ngp * Everything done except novel view testing code * Fixing some nitpicks * Debugging dataloader * merged master * WOP. integrating pipeline with configs and notebook demo * removing model folder * a lot of name changing from Graph to Model * in progress... changing trainer to use pipeline... broken right now * pipeline code runs * switched to val dataset * makes run_eval.py work * updated configs for pipeline change * fixing test_run_train script * fixing black and linting * fix docs * small updates to trainer Co-authored-by: Jake Austin <jake-austin@berkeley.edu>
240 lines
12 KiB
Plaintext
240 lines
12 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Creating pipelines\n",
|
|
"\n",
|
|
"Here we explain how to create custom pipelines in nerfactory. Pipelines are composed of two components, namely a Dataloader and a Model."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"The autoreload extension is already loaded. To reload it, use:\n",
|
|
" %reload_ext autoreload\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# HIDDEN\n",
|
|
"%load_ext autoreload\n",
|
|
"%autoreload 2"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from typing import Dict\n",
|
|
"import torch\n",
|
|
"\n",
|
|
"from nerfactory.dataloaders.base import Dataloader\n",
|
|
"from nerfactory.models.base import Model\n",
|
|
"from nerfactory.pipelines.base import Pipeline\n",
|
|
"\n",
|
|
"from nerfactory.models.instant_ngp import NGPModel\n",
|
|
"from nerfactory.cameras.rays import RayBundle\n",
|
|
"\n",
|
|
"\n",
|
|
"class SemanticNGPModel(Model):\n",
|
|
" \"\"\"An instant ngp model modified slightly to output semantics.\"\"\"\n",
|
|
"\n",
|
|
" def __init__(self):\n",
|
|
" self.ngp_model = NGPModel()\n",
|
|
"\n",
|
|
" def get_outputs(self, ray_bundle: RayBundle) -> Dict[str, torch.Tensor]:\n",
|
|
" outputs = self.ngp_model.forward(ray_bundle)\n",
|
|
" outputs[\"semantics\"] = torch.rand_like(outputs[\"rgb\"])\n",
|
|
" return outputs\n",
|
|
"\n",
|
|
"\n",
|
|
"class CustomPipeline(Pipeline):\n",
|
|
" \"\"\"The Instant NGP pipeline.\"\"\"\n",
|
|
"\n",
|
|
" dataloader: Dataloader\n",
|
|
" model: Model\n",
|
|
"\n",
|
|
" def get_train_loss_dict(self):\n",
|
|
" rays, batch = self.dataloader_train_iter.next()\n",
|
|
" # TODO: do something here\n",
|
|
" accumulated_color, _, _, mask = self.model(rays, batch)\n",
|
|
" masked_batch = get_masked_dict(batch, mask)\n",
|
|
" loss_dict = self.model.get_loss_dict(accumulated_color, masked_batch, mask)\n",
|
|
" return loss_dict\n",
|
|
"\n",
|
|
" def get_eval_loss_dict(self):\n",
|
|
" rays, batch = self.dataloader_eval_iter.next()\n",
|
|
" # TODO: do something here\n",
|
|
" accumulated_color, _, _, mask = self.model(rays, batch)\n",
|
|
" masked_batch = get_masked_dict(batch, mask)\n",
|
|
" loss_dict = self.model.get_loss_dict(accumulated_color, masked_batch, mask)\n",
|
|
" return loss_dict\n",
|
|
"\n",
|
|
" def test_image_outputs(self) -> None:\n",
|
|
" \"\"\"Log the test image outputs\"\"\"\n",
|
|
" # camera = Camera(self.dataloader.eval_datasetinputs.camera_to_world[0])\n",
|
|
" pass"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"ename": "NotImplementedError",
|
|
"evalue": "",
|
|
"output_type": "error",
|
|
"traceback": [
|
|
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
|
"\u001b[0;31mNotImplementedError\u001b[0m Traceback (most recent call last)",
|
|
"\u001b[1;32m/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb Cell 4\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2Bbirman/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb#W3sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m dataloader \u001b[39m=\u001b[39m Dataloader(use_train\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m, use_eval\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bbirman/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb#W3sdnNjb2RlLXJlbW90ZQ%3D%3D?line=1'>2</a>\u001b[0m model \u001b[39m=\u001b[39m SemanticNGPModel()\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bbirman/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb#W3sdnNjb2RlLXJlbW90ZQ%3D%3D?line=2'>3</a>\u001b[0m pipeline \u001b[39m=\u001b[39m CustomPipeline(dataloader\u001b[39m=\u001b[39mdataloader, model\u001b[39m=\u001b[39mmodel)\n",
|
|
"File \u001b[0;32m/shared/ethanweber/nerfactory/nerfactory/dataloaders/base.py:95\u001b[0m, in \u001b[0;36mDataloader.__init__\u001b[0;34m(self, use_train, use_eval)\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[39massert\u001b[39;00m use_train \u001b[39mor\u001b[39;00m use_eval\n\u001b[1;32m 94\u001b[0m \u001b[39mif\u001b[39;00m use_train:\n\u001b[0;32m---> 95\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msetup_train()\n\u001b[1;32m 96\u001b[0m \u001b[39mif\u001b[39;00m use_eval:\n\u001b[1;32m 97\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39msetup_eval()\n",
|
|
"File \u001b[0;32m/shared/ethanweber/nerfactory/nerfactory/dataloaders/base.py:116\u001b[0m, in \u001b[0;36mDataloader.setup_train\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[39m@abstractmethod\u001b[39m\n\u001b[1;32m 114\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39msetup_train\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 115\u001b[0m \u001b[39m\"\"\"Sets up the dataloader for training\"\"\"\u001b[39;00m\n\u001b[0;32m--> 116\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mNotImplementedError\u001b[39;00m\n",
|
|
"\u001b[0;31mNotImplementedError\u001b[0m: "
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"dataloader = Dataloader(use_train=True, use_eval=True)\n",
|
|
"model = SemanticNGPModel()\n",
|
|
"pipeline = CustomPipeline(dataloader=dataloader, model=model)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Creating pipelines from a config\n",
|
|
"\n",
|
|
"Now we show how to create a pipeline from a config, which has the following form:\n",
|
|
"\n",
|
|
"```python\n",
|
|
"@dataclass\n",
|
|
"class PipelineConfig:\n",
|
|
" \"\"\"Configuration for pipeline instantiation\"\"\"\n",
|
|
"\n",
|
|
" _target_: str = MISSING\n",
|
|
" dataloader_config: DataloaderConfig = MISSING\n",
|
|
" graph_config: GraphConfig = MISSING\n",
|
|
"```\n",
|
|
"\n",
|
|
"See [nerfactory/utils/config.py](nerfactory/utils/config.py) for more details. In this example, we will simply load from an existing configuration from [configs/graph_instant_ngp.yaml](configs/graph_instant_ngp.yaml)."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"{'_target_': 'nerfactory.pipelines.base.Pipeline', 'dataloader': {'_target_': 'nerfactory.dataloaders.base.VanillaDataloader', 'train_dataset': {'_target_': 'nerfactory.dataloaders.datasets.Blender', 'data_directory': 'data/blender/lego', 'alpha_color': 'white', 'downscale_factor': 1}, 'train_num_rays_per_batch': 8192, 'eval_dataset': {'_target_': 'nerfactory.dataloaders.datasets.Blender', 'data_directory': 'data/blender/lego', 'alpha_color': 'white', 'downscale_factor': 1}, 'eval_num_rays_per_chunk': 4096, 'eval_num_rays_per_chunk': 8192}, 'model': {'_target_': 'nerfactory.models.instant_ngp.NGPModel', 'enable_collider': False, 'collider_config': {'_target_': 'nerfactory.models.modules.scene_colliders.NearFarCollider', 'near_plane': 2.0, 'far_plane': 6.0}, 'num_coarse_samples': 64, 'num_importance_samples': 128, 'loss_coefficients': {'rgb_loss_coarse': 1.0, 'rgb_loss_fine': 1.0, 'rgb_loss': 1.0}, 'enable_density_field': True, 'density_field_config': {'_target_': 'nerfactory.fields.density_fields.density_grid.DensityGrid', 'center': 0.0, 'base_scale': 3.0, 'num_cascades': 1, 'resolution': 128, 'update_every_num_iters': 16}, 'field_implementation': 'tcnn'}}\n",
|
|
"----------------------------------------------------\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"/shared/ethanweber/miniconda3/envs/nerfactory/lib/python3.8/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and will be removed in 0.15, please use 'weights' instead.\n",
|
|
" warnings.warn(\n",
|
|
"/shared/ethanweber/miniconda3/envs/nerfactory/lib/python3.8/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and will be removed in 0.15. The current behavior is equivalent to passing `weights=AlexNet_Weights.IMAGENET1K_V1`. You can also use `weights=AlexNet_Weights.DEFAULT` to get the most up-to-date weights.\n",
|
|
" warnings.warn(msg)\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import pprint\n",
|
|
"import hydra\n",
|
|
"\n",
|
|
"hydra.core.global_hydra.GlobalHydra.instance().clear()\n",
|
|
"from hydra import compose, initialize\n",
|
|
"\n",
|
|
"initialize(version_base=\"1.2\", config_path=\"../configs/\")\n",
|
|
"config_name = \"graph_instant_ngp.yaml\"\n",
|
|
"config = compose(config_name)\n",
|
|
"pipeline_config = config.pipeline\n",
|
|
"pprint.pprint(pipeline_config)\n",
|
|
"print(\"----------------------------------------------------\")\n",
|
|
"\n",
|
|
"from nerfactory.pipelines.base import setup_pipeline\n",
|
|
"\n",
|
|
"pipeline = setup_pipeline(pipeline_config, device=\"cuda\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"ename": "AttributeError",
|
|
"evalue": "'IterableWrapper' object has no attribute 'next'",
|
|
"output_type": "error",
|
|
"traceback": [
|
|
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
|
"\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
|
|
"\u001b[1;32m/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb Cell 8\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2Bbirman/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb#X13sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m pipeline\u001b[39m.\u001b[39;49mget_train_loss_dict()\n",
|
|
"File \u001b[0;32m/shared/ethanweber/nerfactory/nerfactory/pipelines/base.py:81\u001b[0m, in \u001b[0;36mPipeline.get_train_loss_dict\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 76\u001b[0m \u001b[39m@abstractmethod\u001b[39m\n\u001b[1;32m 77\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mget_train_loss_dict\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 78\u001b[0m \u001b[39m\"\"\"This function gets your training loss dict. This will be responsible for\u001b[39;00m\n\u001b[1;32m 79\u001b[0m \u001b[39m getting the next batch of data from the dataloader and interfacing with the\u001b[39;00m\n\u001b[1;32m 80\u001b[0m \u001b[39m Model class, feeding the data to the model's forward function.\"\"\"\u001b[39;00m\n\u001b[0;32m---> 81\u001b[0m rays, batch \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdataloader_train_iter\u001b[39m.\u001b[39;49mnext()\n\u001b[1;32m 82\u001b[0m accumulated_color, _, _, mask \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel(rays, batch)\n\u001b[1;32m 83\u001b[0m masked_batch \u001b[39m=\u001b[39m get_masked_dict(batch, mask)\n",
|
|
"\u001b[0;31mAttributeError\u001b[0m: 'IterableWrapper' object has no attribute 'next'"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"pipeline.get_train_loss_dict()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3.8.13 ('nerfactory')",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.8.13"
|
|
},
|
|
"orig_nbformat": 4,
|
|
"vscode": {
|
|
"interpreter": {
|
|
"hash": "e691f033e0f2f1b9c0a11e7e81375a1e27aec87a71fd1b1eaa545f7a5e61b3f6"
|
|
}
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
} |