From ce6b9bdd5eeec18e783ff9d65c9a716e91530dc4 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Fri, 28 Jul 2023 11:18:07 -0700 Subject: [PATCH 1/5] Update index.md (#2279) --- docs/index.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/index.md b/docs/index.md index 6e78a1668..0531abf6b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -200,10 +200,11 @@ If you use this library or find the documentation useful for your research, plea ## Maintainers -| | Nerfstudio Discord | Affiliation | -| -------------------------------------------- | ------------------ | ------------------------------------ | -| [Justin Kerr](https://kerrj.github.io/) | justin.kerr | UC Berkeley | -| [Jonáš Kulhánek](https://jkulhanek.com/) | jkulhanek | Czech Technical University in Prague | -| [Matt Tancik](https://www.matthewtancik.com) | tancik | UC Berkeley | -| [Ethan Weber](https://ethanweber.me/) | ethanweber | UC Berkeley | -| [Brent Yi](https://github.com/brentyi) | brent | UC Berkeley | +| | Nerfstudio Discord | Affiliation | +| ----------------------------------------------- | ------------------ | ------------------------------------ | +| [Justin Kerr](https://kerrj.github.io/) | justin.kerr | UC Berkeley | +| [Jonáš Kulhánek](https://jkulhanek.com/) | jkulhanek | Czech Technical University in Prague | +| [Matt Tancik](https://www.matthewtancik.com) | tancik | Luma AI | +| [Matias Turkulainen](https://maturk.github.io/) | maturk | ETH Zurich | +| [Ethan Weber](https://ethanweber.me/) | ethanweber | UC Berkeley | +| [Brent Yi](https://github.com/brentyi) | brent | UC Berkeley | From e6cd95109b74a43541f50bdb8f3f8c5295df94df Mon Sep 17 00:00:00 2001 From: Fernando Rivas-Manzaneque <98962882+Frivas97@users.noreply.github.com> Date: Mon, 31 Jul 2023 15:34:38 +0100 Subject: [PATCH 2/5] Moved Volinga as an external method (#2282) * Moved Volinga as an external method * Update external_methods.py for black * Formatted with black --------- Co-authored-by: jperezh --- docs/extensions/unreal_engine.md | 7 +---- nerfstudio/configs/external_methods.py | 18 +++++++++++ nerfstudio/configs/method_configs.py | 42 -------------------------- tests/test_train.py | 1 - 4 files changed, 19 insertions(+), 49 deletions(-) diff --git a/docs/extensions/unreal_engine.md b/docs/extensions/unreal_engine.md index f533f2a56..1fa523a0d 100644 --- a/docs/extensions/unreal_engine.md +++ b/docs/extensions/unreal_engine.md @@ -10,14 +10,9 @@ NeRFStudio models can be used in Unreal Engine if they are converted to an NVOL file. NVOL is a new standard file format to store NeRFs in a fast and efficient way. NVOL files can be obtained from NeRFStudio checkpoints files (.ckpt) using the [Volinga Suite](https://volinga.ai/). -:::{admonition} Note -:class: warning -Volinga Suite is not yet publicly available. If you are interested in converting your models to NVOL, sign up for the beta at [volinga.ai](https://volinga.ai). -::: - ## Exporting your model to NVOL -Currently NVOL file only supports Volinga model (which is based on nerfacto). You can train your model using the following command: +Currently NVOL file only supports Volinga model (which is based on nerfacto). To use Volinga model you will need to install [Volinga extension for NeRFStudio](https://github.com/Volinga/volinga-model). You can train your model using the following command: ```bash ns-train volinga --data /path/to/your/data --vis viewer diff --git a/nerfstudio/configs/external_methods.py b/nerfstudio/configs/external_methods.py index 4b7433969..6a0c78b09 100644 --- a/nerfstudio/configs/external_methods.py +++ b/nerfstudio/configs/external_methods.py @@ -121,6 +121,24 @@ To enable NeRFPlayer, you must install it first by running: ) ) +# Volinga +external_methods.append( + ExternalMethod( + """[bold yellow]Volinga[/bold yellow] +For more information visit: https://docs.nerf.studio/en/latest/extensions/unreal_engine.html + +To enable Volinga, you must install it first by running: + [grey]pip install git+https://github.com/Volinga/volinga-model[/grey]""", + configurations=[ + ( + "volinga", + "Real-time rendering model from Volinga. Directly exportable to NVOL format at https://volinga.ai/", + ), + ], + pip_package="git+https://github.com/Volinga/volinga-model", + ) +) + @dataclass class ExternalMethodTrainerConfig(TrainerConfig): diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index 428f91d01..72e4f20c8 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -67,7 +67,6 @@ method_configs: Dict[str, TrainerConfig] = {} descriptions = { "nerfacto": "Recommended real-time model tuned for real captures. This model will be continually updated.", "depth-nerfacto": "Nerfacto with depth supervision.", - "volinga": "Real-time rendering model from Volinga. Directly exportable to NVOL format at https://volinga.ai/", "instant-ngp": "Implementation of Instant-NGP. Recommended real-time model for unbounded scenes.", "instant-ngp-bounded": "Implementation of Instant-NGP. Recommended for bounded real and synthetic scenes", "mipnerf": "High quality model for bounded scenes. (slow)", @@ -233,47 +232,6 @@ method_configs["depth-nerfacto"] = TrainerConfig( vis="viewer", ) -method_configs["volinga"] = TrainerConfig( - method_name="volinga", - steps_per_eval_batch=500, - steps_per_save=2000, - max_num_iterations=30000, - mixed_precision=True, - pipeline=VanillaPipelineConfig( - datamanager=VanillaDataManagerConfig( - dataparser=NerfstudioDataParserConfig(), - train_num_rays_per_batch=4096, - eval_num_rays_per_batch=4096, - camera_optimizer=CameraOptimizerConfig( - mode="SO3xR3", optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2) - ), - ), - model=NerfactoModelConfig( - eval_num_rays_per_chunk=1 << 15, - hidden_dim=32, - hidden_dim_color=32, - hidden_dim_transient=32, - num_nerf_samples_per_ray=24, - proposal_net_args_list=[ - {"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 128, "use_linear": True}, - {"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 256, "use_linear": True}, - ], - ), - ), - optimizers={ - "proposal_networks": { - "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), - "scheduler": None, - }, - "fields": { - "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), - "scheduler": None, - }, - }, - viewer=ViewerConfig(num_rays_per_chunk=1 << 15), - vis="viewer", -) - method_configs["instant-ngp"] = TrainerConfig( method_name="instant-ngp", steps_per_eval_batch=500, diff --git a/tests/test_train.py b/tests/test_train.py index d5d25c141..9b95068f9 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -21,7 +21,6 @@ BLACKLIST = [ "instant-ngp", "instant-ngp-bounded", "nerfacto-big", - "volinga", "phototourism", "depth-nerfacto", "neus", From 7e8d67f8a871a2d8cdca7378b2e6f66eade8ce56 Mon Sep 17 00:00:00 2001 From: Sasha Sax Date: Tue, 1 Aug 2023 09:03:01 -0700 Subject: [PATCH 3/5] Add depth_unit_scale_factor to sdfstudio_dataparser (#2285) Depth supervision for `depth-nerfacto` was added in #1173. This patch extends that support to `sdfstudio` datasets ([as originally intended](https://github.com/nerfstudio-project/nerfstudio/issues/1173#issuecomment-1494633338_)) by adding `depth_unit_scale_factor` to `sdfstudio` metadata. --- nerfstudio/data/dataparsers/sdfstudio_dataparser.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nerfstudio/data/dataparsers/sdfstudio_dataparser.py b/nerfstudio/data/dataparsers/sdfstudio_dataparser.py index 85684aeb8..430a8aca8 100644 --- a/nerfstudio/data/dataparsers/sdfstudio_dataparser.py +++ b/nerfstudio/data/dataparsers/sdfstudio_dataparser.py @@ -42,6 +42,8 @@ class SDFStudioDataParserConfig(DataParserConfig): """Directory specifying location of data.""" include_mono_prior: bool = False """whether or not to load monocular depth and normal """ + depth_unit_scale_factor: float = 1e-3 + """Scales the depth values to meters. Default value is 0.001 for a millimeter to meter conversion.""" include_foreground_mask: bool = False """whether or not to load foreground mask""" downscale_factor: int = 1 @@ -153,6 +155,7 @@ class SDFStudio(DataParser): # required for normal maps, these are in colmap format so they require c2w before conversion "camera_to_worlds": c2w_colmap if len(c2w_colmap) > 0 else None, "include_mono_prior": self.config.include_mono_prior, + "depth_unit_scale_factor": self.config.depth_unit_scale_factor, }, ) return dataparser_outputs From 98d36ed7d96898baac2a2be83dc758e53742fbd8 Mon Sep 17 00:00:00 2001 From: AX-I <36831950+AX-I@users.noreply.github.com> Date: Tue, 1 Aug 2023 14:12:30 -0400 Subject: [PATCH 4/5] Option to process images on GPU for speed at the expense of memory (#2288) * Fix pixel sampler cpu bottleneck cpu usage reduced 1700% -> 100%, and even more if using multi-gpu training * Convert to config option * Change optional to bool --- nerfstudio/data/datamanagers/base_datamanager.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index e48e3621a..b3f16af1c 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -115,8 +115,10 @@ class DataManagerConfig(InstantiateConfig): """Source of data, may not be used by all models.""" camera_optimizer: Optional[CameraOptimizerConfig] = None """Specifies the camera pose optimizer used during training. Helpful if poses are noisy.""" - masks_on_gpu: Optional[bool] = None + masks_on_gpu: bool = False """Process masks on GPU for speed at the expense of memory, if True.""" + images_on_gpu: bool = False + """Process images on GPU for speed at the expense of memory, if True.""" class DataManager(nn.Module): @@ -403,6 +405,8 @@ class VanillaDataManager(DataManager, Generic[TDataset]): self.exclude_batch_keys_from_device = self.train_dataset.exclude_batch_keys_from_device if self.config.masks_on_gpu is True: self.exclude_batch_keys_from_device.remove("mask") + if self.config.images_on_gpu is True: + self.exclude_batch_keys_from_device.remove("image") if self.train_dataparser_outputs is not None: cameras = self.train_dataparser_outputs.cameras From dab39c25b4bfc499a44a094abd0c47c112e84c16 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 1 Aug 2023 13:35:22 -0700 Subject: [PATCH 5/5] More robust install instructions (#2277) Add cuda-toolkit install to readme/docs --- README.md | 9 ++++++-- docs/quickstart/installation.md | 41 ++++++++++++++++++--------------- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index ef9b1ed89..290a20735 100644 --- a/README.md +++ b/README.md @@ -92,17 +92,20 @@ Nerfstudio requires `python >= 3.8`. We recommend using conda to manage dependen ```bash conda create --name nerfstudio -y python=3.8 conda activate nerfstudio -python -m pip install --upgrade pip +pip install --upgrade pip ``` ### Dependencies -Install pytorch with CUDA (this repo has been tested with CUDA 11.7 and CUDA 11.8) and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn) +Install PyTorch with CUDA (this repo has been tested with CUDA 11.7 and CUDA 11.8) and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn). +`cuda-toolkit` is required for building `tiny-cuda-nn`. For CUDA 11.7: ```bash pip install torch==2.0.1+cu117 torchvision==0.15.2+cu117 --extra-index-url https://download.pytorch.org/whl/cu117 + +conda install -c "nvidia/label/cuda-11.7.1" cuda-toolkit pip install ninja git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch ``` @@ -110,6 +113,8 @@ For CUDA 11.8: ```bash pip install torch==2.0.1+cu118 torchvision==0.15.2+cu118 --extra-index-url https://download.pytorch.org/whl/cu118 + +conda install -c "nvidia/label/cuda-11.8.0" cuda-toolkit pip install ninja git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch ``` diff --git a/docs/quickstart/installation.md b/docs/quickstart/installation.md index 203d6e35d..30d75ebc3 100644 --- a/docs/quickstart/installation.md +++ b/docs/quickstart/installation.md @@ -33,44 +33,47 @@ python -m pip install --upgrade pip (pytorch)= -### pytorch +### PyTorch -::::{tab-set} -:::{tab-item} Torch 2.0.1 with CUDA 11.8 - -- To install 2.0.1 with CUDA 11.8: - -Note that if a pytorch version prior to 2.0.1 is installed, +Note that if a PyTorch version prior to 2.0.1 is installed, the previous version of pytorch, functorch, and tiny-cuda-nn should be uninstalled. ```bash pip uninstall torch torchvision functorch tinycudann ``` -Install pytorch 2.0.1 with CUDA and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn) +::::{tab-set} +:::{tab-item} Torch 2.0.1 with CUDA 11.8 + +Install PyTorch 2.0.1 with CUDA 11.8: ```bash pip install torch==2.0.1+cu118 torchvision==0.15.2+cu118 --extra-index-url https://download.pytorch.org/whl/cu118 ``` +To build the necessary CUDA extensions, `cuda-toolkit` is also required. We +recommend installing with conda: + +```bash +conda install -c "nvidia/label/cuda-11.7.1" cuda-toolkit +``` + ::: :::{tab-item} Torch 2.0.1 with CUDA 11.7 -- To install 2.0.1 with CUDA 11.7: - -Note that if a pytorch version prior to 2.0.1 is installed, -the previous version of pytorch, functorch, and tiny-cuda-nn should be uninstalled. - -```bash -pip uninstall torch torchvision functorch tinycudann -``` - -Install pytorch 2.0.1 with CUDA and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn) +Install PyTorch 2.0.1 with CUDA 11.7: ```bash pip install torch==2.0.1+cu117 torchvision==0.15.2+cu117 --extra-index-url https://download.pytorch.org/whl/cu117 ``` +To build the necessary CUDA extensions, `cuda-toolkit` is also required. We +recommend installing with conda: + +```bash +conda install -c "nvidia/label/cuda-11.8.0" cuda-toolkit +``` + ::: :::: @@ -258,7 +261,7 @@ While installing tiny-cuda, you run into: `The detected CUDA version mismatches **Solution**: -Reinstall pytorch with the correct CUDA version. +Reinstall PyTorch with the correct CUDA version. See [pytorch](pytorch) under Dependencies, above.