mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-19 14:23:57 +08:00
Reorganizing docs for pipeline restructure (#279)
* starting docs for pipelines * working on an example * clearing output * starting to implement config changes and add documentation * removing diff * format * templated pipeline notebook a bit * black
This commit is contained in:
1 parent
cdacfdaf2e
commit
6879a8a9bd
29 files changed
+619
-416
No files matched your search
@@ -1,240 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Creating pipelines\n",
|
||||
"\n",
|
||||
"Here we explain how to create custom pipelines in nerfactory. Pipelines are composed of two components, namely a Dataloader and a Model."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"The autoreload extension is already loaded. To reload it, use:\n",
|
||||
" %reload_ext autoreload\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# HIDDEN\n",
|
||||
"%load_ext autoreload\n",
|
||||
"%autoreload 2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from typing import Dict\n",
|
||||
"import torch\n",
|
||||
"\n",
|
||||
"from nerfactory.dataloaders.base import Dataloader\n",
|
||||
"from nerfactory.models.base import Model\n",
|
||||
"from nerfactory.pipelines.base import Pipeline\n",
|
||||
"\n",
|
||||
"from nerfactory.models.instant_ngp import NGPModel\n",
|
||||
"from nerfactory.cameras.rays import RayBundle\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class SemanticNGPModel(Model):\n",
|
||||
" \"\"\"An instant ngp model modified slightly to output semantics.\"\"\"\n",
|
||||
"\n",
|
||||
" def __init__(self):\n",
|
||||
" self.ngp_model = NGPModel()\n",
|
||||
"\n",
|
||||
" def get_outputs(self, ray_bundle: RayBundle) -> Dict[str, torch.Tensor]:\n",
|
||||
" outputs = self.ngp_model.forward(ray_bundle)\n",
|
||||
" outputs[\"semantics\"] = torch.rand_like(outputs[\"rgb\"])\n",
|
||||
" return outputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class CustomPipeline(Pipeline):\n",
|
||||
" \"\"\"The Instant NGP pipeline.\"\"\"\n",
|
||||
"\n",
|
||||
" dataloader: Dataloader\n",
|
||||
" model: Model\n",
|
||||
"\n",
|
||||
" def get_train_loss_dict(self):\n",
|
||||
" rays, batch = self.dataloader_train_iter.next()\n",
|
||||
" # TODO: do something here\n",
|
||||
" accumulated_color, _, _, mask = self.model(rays, batch)\n",
|
||||
" masked_batch = get_masked_dict(batch, mask)\n",
|
||||
" loss_dict = self.model.get_loss_dict(accumulated_color, masked_batch, mask)\n",
|
||||
" return loss_dict\n",
|
||||
"\n",
|
||||
" def get_eval_loss_dict(self):\n",
|
||||
" rays, batch = self.dataloader_eval_iter.next()\n",
|
||||
" # TODO: do something here\n",
|
||||
" accumulated_color, _, _, mask = self.model(rays, batch)\n",
|
||||
" masked_batch = get_masked_dict(batch, mask)\n",
|
||||
" loss_dict = self.model.get_loss_dict(accumulated_color, masked_batch, mask)\n",
|
||||
" return loss_dict\n",
|
||||
"\n",
|
||||
" def test_image_outputs(self) -> None:\n",
|
||||
" \"\"\"Log the test image outputs\"\"\"\n",
|
||||
" # camera = Camera(self.dataloader.eval_datasetinputs.camera_to_world[0])\n",
|
||||
" pass"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "NotImplementedError",
|
||||
"evalue": "",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mNotImplementedError\u001b[0m Traceback (most recent call last)",
|
||||
"\u001b[1;32m/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb Cell 4\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2Bbirman/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb#W3sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m dataloader \u001b[39m=\u001b[39m Dataloader(use_train\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m, use_eval\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bbirman/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb#W3sdnNjb2RlLXJlbW90ZQ%3D%3D?line=1'>2</a>\u001b[0m model \u001b[39m=\u001b[39m SemanticNGPModel()\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bbirman/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb#W3sdnNjb2RlLXJlbW90ZQ%3D%3D?line=2'>3</a>\u001b[0m pipeline \u001b[39m=\u001b[39m CustomPipeline(dataloader\u001b[39m=\u001b[39mdataloader, model\u001b[39m=\u001b[39mmodel)\n",
|
||||
"File \u001b[0;32m/shared/ethanweber/nerfactory/nerfactory/dataloaders/base.py:95\u001b[0m, in \u001b[0;36mDataloader.__init__\u001b[0;34m(self, use_train, use_eval)\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[39massert\u001b[39;00m use_train \u001b[39mor\u001b[39;00m use_eval\n\u001b[1;32m 94\u001b[0m \u001b[39mif\u001b[39;00m use_train:\n\u001b[0;32m---> 95\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msetup_train()\n\u001b[1;32m 96\u001b[0m \u001b[39mif\u001b[39;00m use_eval:\n\u001b[1;32m 97\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39msetup_eval()\n",
|
||||
"File \u001b[0;32m/shared/ethanweber/nerfactory/nerfactory/dataloaders/base.py:116\u001b[0m, in \u001b[0;36mDataloader.setup_train\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[39m@abstractmethod\u001b[39m\n\u001b[1;32m 114\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39msetup_train\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 115\u001b[0m \u001b[39m\"\"\"Sets up the dataloader for training\"\"\"\u001b[39;00m\n\u001b[0;32m--> 116\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mNotImplementedError\u001b[39;00m\n",
|
||||
"\u001b[0;31mNotImplementedError\u001b[0m: "
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"dataloader = Dataloader(use_train=True, use_eval=True)\n",
|
||||
"model = SemanticNGPModel()\n",
|
||||
"pipeline = CustomPipeline(dataloader=dataloader, model=model)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Creating pipelines from a config\n",
|
||||
"\n",
|
||||
"Now we show how to create a pipeline from a config, which has the following form:\n",
|
||||
"\n",
|
||||
"```python\n",
|
||||
"@dataclass\n",
|
||||
"class PipelineConfig:\n",
|
||||
" \"\"\"Configuration for pipeline instantiation\"\"\"\n",
|
||||
"\n",
|
||||
" _target_: str = MISSING\n",
|
||||
" dataloader_config: DataloaderConfig = MISSING\n",
|
||||
" graph_config: GraphConfig = MISSING\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"See [nerfactory/utils/config.py](nerfactory/utils/config.py) for more details. In this example, we will simply load from an existing configuration from [configs/graph_instant_ngp.yaml](configs/graph_instant_ngp.yaml)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"{'_target_': 'nerfactory.pipelines.base.Pipeline', 'dataloader': {'_target_': 'nerfactory.dataloaders.base.VanillaDataloader', 'train_dataset': {'_target_': 'nerfactory.dataloaders.datasets.Blender', 'data_directory': 'data/blender/lego', 'alpha_color': 'white', 'downscale_factor': 1}, 'train_num_rays_per_batch': 8192, 'eval_dataset': {'_target_': 'nerfactory.dataloaders.datasets.Blender', 'data_directory': 'data/blender/lego', 'alpha_color': 'white', 'downscale_factor': 1}, 'eval_num_rays_per_chunk': 4096, 'eval_num_rays_per_chunk': 8192}, 'model': {'_target_': 'nerfactory.models.instant_ngp.NGPModel', 'enable_collider': False, 'collider_config': {'_target_': 'nerfactory.models.modules.scene_colliders.NearFarCollider', 'near_plane': 2.0, 'far_plane': 6.0}, 'num_coarse_samples': 64, 'num_importance_samples': 128, 'loss_coefficients': {'rgb_loss_coarse': 1.0, 'rgb_loss_fine': 1.0, 'rgb_loss': 1.0}, 'enable_density_field': True, 'density_field_config': {'_target_': 'nerfactory.fields.density_fields.density_grid.DensityGrid', 'center': 0.0, 'base_scale': 3.0, 'num_cascades': 1, 'resolution': 128, 'update_every_num_iters': 16}, 'field_implementation': 'tcnn'}}\n",
|
||||
"----------------------------------------------------\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/shared/ethanweber/miniconda3/envs/nerfactory/lib/python3.8/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and will be removed in 0.15, please use 'weights' instead.\n",
|
||||
" warnings.warn(\n",
|
||||
"/shared/ethanweber/miniconda3/envs/nerfactory/lib/python3.8/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and will be removed in 0.15. The current behavior is equivalent to passing `weights=AlexNet_Weights.IMAGENET1K_V1`. You can also use `weights=AlexNet_Weights.DEFAULT` to get the most up-to-date weights.\n",
|
||||
" warnings.warn(msg)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import pprint\n",
|
||||
"import hydra\n",
|
||||
"\n",
|
||||
"hydra.core.global_hydra.GlobalHydra.instance().clear()\n",
|
||||
"from hydra import compose, initialize\n",
|
||||
"\n",
|
||||
"initialize(version_base=\"1.2\", config_path=\"../configs/\")\n",
|
||||
"config_name = \"graph_instant_ngp.yaml\"\n",
|
||||
"config = compose(config_name)\n",
|
||||
"pipeline_config = config.pipeline\n",
|
||||
"pprint.pprint(pipeline_config)\n",
|
||||
"print(\"----------------------------------------------------\")\n",
|
||||
"\n",
|
||||
"from nerfactory.pipelines.base import setup_pipeline\n",
|
||||
"\n",
|
||||
"pipeline = setup_pipeline(pipeline_config, device=\"cuda\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "AttributeError",
|
||||
"evalue": "'IterableWrapper' object has no attribute 'next'",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
|
||||
"\u001b[1;32m/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb Cell 8\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2Bbirman/shared/ethanweber/nerfactory/notebooks/creating_pipelines.ipynb#X13sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m pipeline\u001b[39m.\u001b[39;49mget_train_loss_dict()\n",
|
||||
"File \u001b[0;32m/shared/ethanweber/nerfactory/nerfactory/pipelines/base.py:81\u001b[0m, in \u001b[0;36mPipeline.get_train_loss_dict\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 76\u001b[0m \u001b[39m@abstractmethod\u001b[39m\n\u001b[1;32m 77\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mget_train_loss_dict\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 78\u001b[0m \u001b[39m\"\"\"This function gets your training loss dict. This will be responsible for\u001b[39;00m\n\u001b[1;32m 79\u001b[0m \u001b[39m getting the next batch of data from the dataloader and interfacing with the\u001b[39;00m\n\u001b[1;32m 80\u001b[0m \u001b[39m Model class, feeding the data to the model's forward function.\"\"\"\u001b[39;00m\n\u001b[0;32m---> 81\u001b[0m rays, batch \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdataloader_train_iter\u001b[39m.\u001b[39;49mnext()\n\u001b[1;32m 82\u001b[0m accumulated_color, _, _, mask \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel(rays, batch)\n\u001b[1;32m 83\u001b[0m masked_batch \u001b[39m=\u001b[39m get_masked_dict(batch, mask)\n",
|
||||
"\u001b[0;31mAttributeError\u001b[0m: 'IterableWrapper' object has no attribute 'next'"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"pipeline.get_train_loss_dict()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.8.13 ('nerfactory')",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.8.13"
|
||||
},
|
||||
"orig_nbformat": 4,
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "e691f033e0f2f1b9c0a11e7e81375a1e27aec87a71fd1b1eaa545f7a5e61b3f6"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Reference in new issue
Block a user