From 1b4b682bceb9c0e81441768d1af959c33e2ac1ea Mon Sep 17 00:00:00 2001 From: Ling Jing <111032642+Jing1Ling@users.noreply.github.com> Date: Tue, 13 Feb 2024 06:10:14 +0800 Subject: [PATCH 1/6] add zipnerf-pytorch as external method (#2850) * add zipnerf-pytorch as external method --- docs/index.md | 1 + docs/nerfology/methods/index.md | 1 + docs/nerfology/methods/zipnerf.md | 33 ++++++++++++++++++++++++++ nerfstudio/configs/external_methods.py | 16 +++++++++++++ 4 files changed, 51 insertions(+) create mode 100644 docs/nerfology/methods/zipnerf.md diff --git a/docs/index.md b/docs/index.md index 685dc635e..fcb62c644 100644 --- a/docs/index.md +++ b/docs/index.md @@ -156,6 +156,7 @@ This documentation is organized into 3 parts: - [Instruct-GS2GS](nerfology/methods/igs2gs.md): Editing 3DGS Scenes with Instructions - [PyNeRF](nerfology/methods/pynerf.md): Pyramidal Neural Radiance Fields - [SeaThru-NeRF](nerfology/methods/seathru_nerf.md): Neural Radiance Field for subsea scenes +- [Zip-NeRF](nerfology/methods/zipnerf.md): Anti-Aliased Grid-Based Neural Radiance Fields **Eager to contribute a method?** We'd love to see you use nerfstudio in implementing new (or even existing) methods! Please view our {ref}`guide` for more details about how to add to this list! diff --git a/docs/nerfology/methods/index.md b/docs/nerfology/methods/index.md index d29477221..5284b6887 100644 --- a/docs/nerfology/methods/index.md +++ b/docs/nerfology/methods/index.md @@ -42,6 +42,7 @@ The following methods are supported in nerfstudio: Instruct-GS2GS PyNeRF SeaThru-NeRF + Zip-NeRF ``` (own_method_docs)= diff --git a/docs/nerfology/methods/zipnerf.md b/docs/nerfology/methods/zipnerf.md new file mode 100644 index 000000000..f435eabb3 --- /dev/null +++ b/docs/nerfology/methods/zipnerf.md @@ -0,0 +1,33 @@ +# Zip-NeRF + +

A pytorch implementation of "Zip-NeRF: Anti-Aliased Grid-Based Neural Radiance Fields"

+ +```{button-link} https://jonbarron.info/zipnerf/ +:color: primary +:outline: +Paper Website +``` +```{button-link} https://github.com/SuLvXiangXin/zipnerf-pytorch +:color: primary +:outline: +Code +``` +### Installation +First, install nerfstudio and its dependencies. Then run: +``` +pip install git+https://github.com/SuLvXiangXin/zipnerf-pytorch#subdirectory=extensions/cuda +pip install git+https://github.com/SuLvXiangXin/zipnerf-pytorch +``` +Finally, install torch_scatter corresponding to your cuda version(https://pytorch-geometric.com/whl/torch-2.0.1%2Bcu118.html). + + +### Running Model + +```bash +ns-train zipnerf --data {DATA_DIR/SCENE} +``` + +## Overview +Zipnerf combines mip-NeRF 360’s overall framework with iNGP’s featurization approach. +Following mip-NeRF, zipnerf assume each pixel corresponds to a cone. Given an interval along the ray, it construct a set of multisamples that approximate the shape of that conical frustum. +Also, it present an alternative loss that, unlike mip-NeRF 360’s interlevel loss, is continuous and smooth with respect to distance along the ray to prevent z-aliasing. \ No newline at end of file diff --git a/nerfstudio/configs/external_methods.py b/nerfstudio/configs/external_methods.py index 7dbae661f..a0982f098 100644 --- a/nerfstudio/configs/external_methods.py +++ b/nerfstudio/configs/external_methods.py @@ -192,6 +192,22 @@ To enable Seathru-NeRF, you must install it first by running: ) ) +# Zip-NeRF +external_methods.append( + ExternalMethod( + """[bold yellow]Zip-NeRF[/bold yellow] +For more information visit https://docs.nerf.studio/nerfology/methods/zipnerf.html + +To enable Zip-NeRF, you must install it first by running: + [grey]pip install git+https://github.com/SuLvXiangXin/zipnerf-pytorch#subdirectory=extensions/cuda + and pip install git+https://github.com/SuLvXiangXin/zipnerf-pytorch[/grey]""", + configurations=[ + ("zipnerf", "A pytorch implementation of 'Zip-NeRF: Anti-Aliased Grid-Based Neural Radiance Fields'") + ], + pip_package="pip install git+https://github.com/SuLvXiangXin/zipnerf-pytorch", + ) +) + @dataclass class ExternalMethodDummyTrainerConfig: From 57fbc07bf1efecdb767259148fe5705ccf12af3f Mon Sep 17 00:00:00 2001 From: Pierre-Alain Langlois <156442943+pa-la@users.noreply.github.com> Date: Mon, 12 Feb 2024 18:31:42 -0500 Subject: [PATCH 2/6] Add transparency carving to splatfacto (#2889) Implemented alpha training in splatfacto --- nerfstudio/models/splatfacto.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/nerfstudio/models/splatfacto.py b/nerfstudio/models/splatfacto.py index a826599f9..faaa2150c 100644 --- a/nerfstudio/models/splatfacto.py +++ b/nerfstudio/models/splatfacto.py @@ -672,7 +672,7 @@ class SplatfactoModel(Model): rgb = background.repeat(int(camera.height.item()), int(camera.width.item()), 1) depth = background.new_ones(*rgb.shape[:2], 1) * 10 accumulation = background.new_zeros(*rgb.shape[:2], 1) - return {"rgb": rgb, "depth": depth, "accumulation": accumulation} + return {"rgb": rgb, "depth": depth, "accumulation": accumulation, "background": background} else: crop_ids = None camera_downscale = self._get_downscale_factor() @@ -740,7 +740,7 @@ class SplatfactoModel(Model): rgb = background.repeat(int(camera.height.item()), int(camera.width.item()), 1) depth = background.new_ones(*rgb.shape[:2], 1) * 10 accumulation = background.new_zeros(*rgb.shape[:2], 1) - return {"rgb": rgb, "depth": depth, "accumulation": accumulation} + return {"rgb": rgb, "depth": depth, "accumulation": accumulation, "background": background} # Important to allow xys grads to populate properly if self.training: @@ -789,7 +789,7 @@ class SplatfactoModel(Model): )[..., 0:1] # type: ignore depth_im = torch.where(alpha > 0, depth_im / alpha, depth_im.detach().max()) - return {"rgb": rgb, "depth": depth_im, "accumulation": alpha} # type: ignore + return {"rgb": rgb, "depth": depth_im, "accumulation": alpha, "background": background} # type: ignore def get_gt_img(self, image: torch.Tensor): """Compute groundtruth image with iteration dependent downscale factor for evaluation purpose @@ -811,6 +811,19 @@ class SplatfactoModel(Model): gt_img = image return gt_img.to(self.device) + def composite_with_background(self, image, background) -> torch.Tensor: + """Composite the ground truth image with a background color when it has an alpha channel. + + Args: + image: the image to composite + background: the background color + """ + if image.shape[2] == 4: + alpha = image[..., -1].unsqueeze(-1).repeat((1, 1, 3)) + return alpha * image[..., :3] + (1 - alpha) * background + else: + return image + def get_metrics_dict(self, outputs, batch) -> Dict[str, torch.Tensor]: """Compute and returns metrics. @@ -818,7 +831,7 @@ class SplatfactoModel(Model): outputs: the output to compute loss dict to batch: ground truth batch corresponding to outputs """ - gt_rgb = self.get_gt_img(batch["image"]) + gt_rgb = self.composite_with_background(self.get_gt_img(batch["image"]), outputs["background"]) metrics_dict = {} predicted_rgb = outputs["rgb"] metrics_dict["psnr"] = self.psnr(predicted_rgb, gt_rgb) @@ -834,7 +847,7 @@ class SplatfactoModel(Model): batch: ground truth batch corresponding to outputs metrics_dict: dictionary of metrics, some of which we can use for loss """ - gt_img = self.get_gt_img(batch["image"]) + gt_img = self.composite_with_background(self.get_gt_img(batch["image"]), outputs["background"]) pred_img = outputs["rgb"] # Set masked part of both ground-truth and rendered image to black. @@ -893,7 +906,7 @@ class SplatfactoModel(Model): Returns: A dictionary of metrics. """ - gt_rgb = self.get_gt_img(batch["image"]) + gt_rgb = self.composite_with_background(self.get_gt_img(batch["image"]), outputs["background"]) d = self._get_downscale_factor() if d > 1: # torchvision can be slow to import, so we do it lazily. From 4f798b23f6c65ef2970145901a0251b61ec8a447 Mon Sep 17 00:00:00 2001 From: Sebastiaan <751205+SharkWipf@users.noreply.github.com> Date: Tue, 13 Feb 2024 19:09:59 +0100 Subject: [PATCH 3/6] Fix an issue where gradient accumulation could not be passed as argument due to a type error. (#2910) --- nerfstudio/engine/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index 68b026c13..9b4d6d8a1 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -82,7 +82,7 @@ class TrainerConfig(ExperimentConfig): """Path to checkpoint file.""" log_gradients: bool = False """Optionally log gradients during training""" - gradient_accumulation_steps: Dict = field(default_factory=lambda: {}) + gradient_accumulation_steps: Dict[str, int] = field(default_factory=lambda: {}) """Number of steps to accumulate gradients over. Contains a mapping of {param_group:num}""" From eb9bcb577d5bc0249eba05f6d11a3b4e53ef8a3d Mon Sep 17 00:00:00 2001 From: "J.Y" <132313008+jb-ye@users.noreply.github.com> Date: Tue, 13 Feb 2024 14:43:04 -0500 Subject: [PATCH 4/6] Implement new rasterization mode using opacity compensation factor (#2888) Implement new rasterization mode for tiny gaussians --- nerfstudio/models/splatfacto.py | 25 +++++++++++++++++++++++-- pyproject.toml | 2 +- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/nerfstudio/models/splatfacto.py b/nerfstudio/models/splatfacto.py index faaa2150c..90bdccd70 100644 --- a/nerfstudio/models/splatfacto.py +++ b/nerfstudio/models/splatfacto.py @@ -157,6 +157,16 @@ class SplatfactoModelConfig(ModelConfig): """ output_depth_during_training: bool = False """If True, output depth during training. Otherwise, only output depth during evaluation.""" + rasterize_mode: Literal["classic", "antialiased"] = "classic" + """ + Classic mode of rendering will use the EWA volume splatting with a [0.3, 0.3] screen space blurring kernel. This + approach is however not suitable to render tiny gaussians at higher or lower resolution than the captured, which + results "aliasing-like" artifacts. The antialiased mode overcomes this limitation by calculating compensation factors + and apply them to the opacities of gaussians to preserve the total integrated density of splats. + + However, PLY exported with antialiased rasterize mode is not compatible with classic mode. Thus many web viewers that + were implemented for classic mode can not render antialiased mode PLY properly without modifications. + """ class SplatfactoModel(Model): @@ -721,7 +731,7 @@ class SplatfactoModel(Model): colors_crop = torch.cat((features_dc_crop[:, None, :], features_rest_crop), dim=1) - self.xys, depths, self.radii, conics, num_tiles_hit, cov3d = project_gaussians( # type: ignore + self.xys, depths, self.radii, conics, comp, num_tiles_hit, cov3d = project_gaussians( # type: ignore means_crop, torch.exp(scales_crop), 1, @@ -736,6 +746,7 @@ class SplatfactoModel(Model): W, tile_bounds, ) # type: ignore + if (self.radii).sum() == 0: rgb = background.repeat(int(camera.height.item()), int(camera.width.item()), 1) depth = background.new_ones(*rgb.shape[:2], 1) * 10 @@ -758,6 +769,16 @@ class SplatfactoModel(Model): # rescale the camera back to original dimensions camera.rescale_output_resolution(camera_downscale) assert (num_tiles_hit > 0).any() # type: ignore + + # apply the compensation of screen space blurring to gaussians + opacities = None + if self.config.rasterize_mode == "antialiased": + opacities = torch.sigmoid(opacities_crop) * comp[:, None] + elif self.config.rasterize_mode == "classic": + opacities = torch.sigmoid(opacities_crop) + else: + raise ValueError("Unknown rasterize_mode: %s", self.config.rasterize_mode) + rgb, alpha = rasterize_gaussians( # type: ignore self.xys, depths, @@ -765,7 +786,7 @@ class SplatfactoModel(Model): conics, num_tiles_hit, # type: ignore rgbs, - torch.sigmoid(opacities_crop), + opacities, H, W, background=background, diff --git a/pyproject.toml b/pyproject.toml index cde1003e2..e2a731f64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,7 @@ dependencies = [ "xatlas", "trimesh>=3.20.2", "timm==0.6.7", - "gsplat==0.1.3", + "gsplat>=0.1.5", "pytorch-msssim", "pathos", "packaging" From 7f9def1ff0b869dcf1d00abe475759de05cdf435 Mon Sep 17 00:00:00 2001 From: Justin Kerr Date: Tue, 13 Feb 2024 12:26:17 -0800 Subject: [PATCH 5/6] Switch default matching method for videos to 'sequential' in COLMAP (#2912) default matching method for video to sequential for COLMAP --- nerfstudio/process_data/video_to_nerfstudio_dataset.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nerfstudio/process_data/video_to_nerfstudio_dataset.py b/nerfstudio/process_data/video_to_nerfstudio_dataset.py index 928752e43..af17e7d6b 100644 --- a/nerfstudio/process_data/video_to_nerfstudio_dataset.py +++ b/nerfstudio/process_data/video_to_nerfstudio_dataset.py @@ -16,6 +16,7 @@ import shutil from dataclasses import dataclass +from typing import Literal from nerfstudio.process_data import equirect_utils, process_data_utils from nerfstudio.process_data.colmap_converter_to_nerfstudio_dataset import ColmapConverterToNerfstudioDataset @@ -36,6 +37,10 @@ class VideoToNerfstudioDataset(ColmapConverterToNerfstudioDataset): """Target number of frames to use per video, results may not be exact.""" percent_radius_crop: float = 1.0 """Create circle crop mask. The radius is the percent of the image diagonal.""" + matching_method: Literal["exhaustive", "sequential", "vocab_tree"] = "sequential" + """Feature matching method to use. Vocab tree is recommended for a balance of speed + and accuracy. Exhaustive is slower but more accurate. Sequential is faster but + should only be used for videos.""" def main(self) -> None: """Process video into a nerfstudio dataset.""" From 45db2bcfabe6e0644a3a45a50ed80a9a685ddc34 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 13 Feb 2024 18:12:18 -0500 Subject: [PATCH 6/6] Call rescale_output_resolution before early return (#2897) * Call rescale_output_resolution before early return Co-authored-by: Chung Min Kim --- nerfstudio/models/splatfacto.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nerfstudio/models/splatfacto.py b/nerfstudio/models/splatfacto.py index 90bdccd70..071a063e3 100644 --- a/nerfstudio/models/splatfacto.py +++ b/nerfstudio/models/splatfacto.py @@ -747,10 +747,14 @@ class SplatfactoModel(Model): tile_bounds, ) # type: ignore + # rescale the camera back to original dimensions before returning + camera.rescale_output_resolution(camera_downscale) + if (self.radii).sum() == 0: - rgb = background.repeat(int(camera.height.item()), int(camera.width.item()), 1) + rgb = background.repeat(H, W, 1) depth = background.new_ones(*rgb.shape[:2], 1) * 10 accumulation = background.new_zeros(*rgb.shape[:2], 1) + return {"rgb": rgb, "depth": depth, "accumulation": accumulation, "background": background} # Important to allow xys grads to populate properly @@ -766,8 +770,6 @@ class SplatfactoModel(Model): else: rgbs = torch.sigmoid(colors_crop[:, 0, :]) - # rescale the camera back to original dimensions - camera.rescale_output_resolution(camera_downscale) assert (num_tiles_hit > 0).any() # type: ignore # apply the compensation of screen space blurring to gaussians