merge conflicts with main

This commit is contained in:
cardiacmangoes committed 2024-02-16 22:33:35 +00:00
commit 0cb3ff77ff
8 files changed
+112 -22

No files matched your search

+1
View File
@@ -156,6 +156,7 @@ This documentation is organized into 3 parts:
- [Instruct-GS2GS](nerfology/methods/igs2gs.md): Editing 3DGS Scenes with Instructions
- [PyNeRF](nerfology/methods/pynerf.md): Pyramidal Neural Radiance Fields
- [SeaThru-NeRF](nerfology/methods/seathru_nerf.md): Neural Radiance Field for subsea scenes
- [Zip-NeRF](nerfology/methods/zipnerf.md): Anti-Aliased Grid-Based Neural Radiance Fields
**Eager to contribute a method?** We'd love to see you use nerfstudio in implementing new (or even existing) methods! Please view our {ref}`guide<own_method_docs>` for more details about how to add to this list!
+1
View File
@@ -42,6 +42,7 @@ The following methods are supported in nerfstudio:
Instruct-GS2GS<igs2gs.md>
PyNeRF<pynerf.md>
SeaThru-NeRF<seathru_nerf.md>
Zip-NeRF<zipnerf.md>
```
(own_method_docs)=
+33
View File
@@ -0,0 +1,33 @@
# Zip-NeRF
<h4>A pytorch implementation of "Zip-NeRF: Anti-Aliased Grid-Based Neural Radiance Fields"</h4>
```{button-link} https://jonbarron.info/zipnerf/
:color: primary
:outline:
Paper Website
```
```{button-link} https://github.com/SuLvXiangXin/zipnerf-pytorch
:color: primary
:outline:
Code
```
### Installation
First, install nerfstudio and its dependencies. Then run:
```
pip install git+https://github.com/SuLvXiangXin/zipnerf-pytorch#subdirectory=extensions/cuda
pip install git+https://github.com/SuLvXiangXin/zipnerf-pytorch
```
Finally, install torch_scatter corresponding to your cuda version(https://pytorch-geometric.com/whl/torch-2.0.1%2Bcu118.html).
### Running Model
```bash
ns-train zipnerf --data {DATA_DIR/SCENE}
```
## Overview
Zipnerf combines mip-NeRF 360’s overall framework with iNGP’s featurization approach.
Following mip-NeRF, zipnerf assume each pixel corresponds to a cone. Given an interval along the ray, it construct a set of multisamples that approximate the shape of that conical frustum.
Also, it present an alternative loss that, unlike mip-NeRF 360’s interlevel loss, is continuous and smooth with respect to distance along the ray to prevent z-aliasing.
+16
View File
@@ -192,6 +192,22 @@ To enable Seathru-NeRF, you must install it first by running:
)
)
# Zip-NeRF
external_methods.append(
ExternalMethod(
"""[bold yellow]Zip-NeRF[/bold yellow]
For more information visit https://docs.nerf.studio/nerfology/methods/zipnerf.html
To enable Zip-NeRF, you must install it first by running:
[grey]pip install git+https://github.com/SuLvXiangXin/zipnerf-pytorch#subdirectory=extensions/cuda
and pip install git+https://github.com/SuLvXiangXin/zipnerf-pytorch[/grey]""",
configurations=[
("zipnerf", "A pytorch implementation of 'Zip-NeRF: Anti-Aliased Grid-Based Neural Radiance Fields'")
],
pip_package="pip install git+https://github.com/SuLvXiangXin/zipnerf-pytorch",
)
)
@dataclass
class ExternalMethodDummyTrainerConfig:
+1 -1
View File
@@ -82,7 +82,7 @@ class TrainerConfig(ExperimentConfig):
"""Path to checkpoint file."""
log_gradients: bool = False
"""Optionally log gradients during training"""
gradient_accumulation_steps: Dict = field(default_factory=lambda: {})
gradient_accumulation_steps: Dict[str, int] = field(default_factory=lambda: {})
"""Number of steps to accumulate gradients over. Contains a mapping of {param_group:num}"""
+54 -20
View File
@@ -157,6 +157,16 @@ class SplatfactoModelConfig(ModelConfig):
"""
output_depth_during_training: bool = False
"""If True, output depth during training. Otherwise, only output depth during evaluation."""
rasterize_mode: Literal["classic", "antialiased"] = "classic"
"""
Classic mode of rendering will use the EWA volume splatting with a [0.3, 0.3] screen space blurring kernel. This
approach is however not suitable to render tiny gaussians at higher or lower resolution than the captured, which
results "aliasing-like" artifacts. The antialiased mode overcomes this limitation by calculating compensation factors
and apply them to the opacities of gaussians to preserve the total integrated density of splats.
However, PLY exported with antialiased rasterize mode is not compatible with classic mode. Thus many web viewers that
were implemented for classic mode can not render antialiased mode PLY properly without modifications.
"""
use_nd: bool = False
"""Whether to rasterize nd values."""
nd_dim: int = 1
@@ -235,7 +245,6 @@ class SplatfactoModel(Model):
) # This color is the same as the default background color in Viser. This would only affect the background color when rendering.
else:
self.background_color = get_color(self.config.background_color)
if self.config.use_nd:
self.nd_values = torch.nn.Parameter(torch.rand(self.num_points, self.config.nd_dim))
@@ -449,9 +458,10 @@ class SplatfactoModel(Model):
dim=0,
)
)
if self.config.use_nd:
self.nd_values = Parameter(torch.cat([self.nd_values.detach(), split_nd_values, dup_nd_values], dim=0))
self.nd_values = Parameter(
torch.cat([self.nd_values.detach(), split_nd_values, dup_nd_values], dim=0)
)
self.opacities = Parameter(torch.cat([self.opacities.detach(), split_opacities, dup_opacities], dim=0))
self.scales = Parameter(torch.cat([self.scales.detach(), split_scales, dup_scales], dim=0))
@@ -701,7 +711,7 @@ class SplatfactoModel(Model):
# get the background color
dims = self.config.nd_dim if self.config.use_nd else 3
if self.training:
if self.training:
if self.config.background_color == "random":
background = torch.rand(3, device=self.device)
nd_background = torch.rand(dims, device=self.device)
@@ -720,7 +730,6 @@ class SplatfactoModel(Model):
background = renderers.BACKGROUND_COLOR_OVERRIDE.to(self.device)
else:
background = self.background_color.to(self.device)
if self.crop_box is not None and not self.training:
crop_ids = self.crop_box.within(self.means).squeeze()
@@ -732,7 +741,7 @@ class SplatfactoModel(Model):
if self.config.use_nd:
out["nd_value"] = nd_background.repeat(int(camera.height.item()), int(camera.width.item()), 1)
out.update({"rgb": rgb, "depth": depth, "accumulation": accumulation})
out.update({"rgb": rgb, "depth": depth, "accumulation": accumulation, "background": background})
return out
else:
crop_ids = None
@@ -782,7 +791,7 @@ class SplatfactoModel(Model):
colors_crop = torch.cat((features_dc_crop[:, None, :], features_rest_crop), dim=1)
self.xys, depths, self.radii, conics, num_tiles_hit, cov3d = project_gaussians( # type: ignore
self.xys, depths, self.radii, conics, comp, num_tiles_hit, cov3d = project_gaussians( # type: ignore
means_crop,
torch.exp(scales_crop),
1,
@@ -797,13 +806,18 @@ class SplatfactoModel(Model):
W,
tile_bounds,
) # type: ignore
# rescale the camera back to original dimensions before returning
camera.rescale_output_resolution(camera_downscale)
if (self.radii).sum() == 0:
rgb = background.repeat(int(camera.height.item()), int(camera.width.item()), 1)
rgb = background.repeat(H, W, 1)
depth = background.new_ones(*rgb.shape[:2], 1) * 10
accumulation = background.new_zeros(*rgb.shape[:2], 1)
if self.config.use_nd:
out["nd_value"] = nd_background.repeat(int(camera.height.item()), int(camera.width.item()), 1)
out.update({"rgb": rgb, "depth": depth, "accumulation": accumulation})
out.update({"rgb": rgb, "depth": depth, "accumulation": accumulation, "background": background})
return out
# Important to allow xys grads to populate properly
@@ -819,9 +833,17 @@ class SplatfactoModel(Model):
else:
rgbs = torch.sigmoid(colors_crop[:, 0, :])
# rescale the camera back to original dimensions
camera.rescale_output_resolution(camera_downscale)
assert (num_tiles_hit > 0).any() # type: ignore
# apply the compensation of screen space blurring to gaussians
opacities = None
if self.config.rasterize_mode == "antialiased":
opacities = torch.sigmoid(opacities_crop) * comp[:, None]
elif self.config.rasterize_mode == "classic":
opacities = torch.sigmoid(opacities_crop)
else:
raise ValueError("Unknown rasterize_mode: %s", self.config.rasterize_mode)
rgb, alpha = rasterize_gaussians( # type: ignore
self.xys,
depths,
@@ -829,7 +851,7 @@ class SplatfactoModel(Model):
conics,
num_tiles_hit, # type: ignore
rgbs,
torch.sigmoid(opacities_crop),
opacities,
H,
W,
background=background,
@@ -852,7 +874,6 @@ class SplatfactoModel(Model):
background=torch.zeros(3, device=self.device),
)[..., 0:1] # type: ignore
depth_im = torch.where(alpha > 0, depth_im / alpha, depth_im.detach().max())
if self.config.use_nd:
nd_values = self.get_nd_values()
nd_value = rasterize_gaussians( # type: ignore
@@ -865,12 +886,12 @@ class SplatfactoModel(Model):
torch.sigmoid(opacities_crop.detach()) if self.config.nd_detach else torch.sigmoid(opacities_crop),
H,
W,
background= nd_background,
)
background=nd_background,
)
out["nd_value"] = nd_value
out.update({"rgb": rgb, "depth": depth_im, "accumulation": alpha})
return out # type: ignore
out.update({"rgb": rgb, "depth": depth_im, "accumulation": alpha, "background": background})
return out # type: ignore
def get_gt_img(self, image: torch.Tensor):
"""Compute groundtruth image with iteration dependent downscale factor for evaluation purpose
@@ -892,6 +913,19 @@ class SplatfactoModel(Model):
gt_img = image
return gt_img.to(self.device)
def composite_with_background(self, image, background) -> torch.Tensor:
"""Composite the ground truth image with a background color when it has an alpha channel.
Args:
image: the image to composite
background: the background color
"""
if image.shape[2] == 4:
alpha = image[..., -1].unsqueeze(-1).repeat((1, 1, 3))
return alpha * image[..., :3] + (1 - alpha) * background
else:
return image
def get_metrics_dict(self, outputs, batch) -> Dict[str, torch.Tensor]:
"""Compute and returns metrics.
@@ -899,7 +933,7 @@ class SplatfactoModel(Model):
outputs: the output to compute loss dict to
batch: ground truth batch corresponding to outputs
"""
gt_rgb = self.get_gt_img(batch["image"])
gt_rgb = self.composite_with_background(self.get_gt_img(batch["image"]), outputs["background"])
metrics_dict = {}
predicted_rgb = outputs["rgb"]
metrics_dict["psnr"] = self.psnr(predicted_rgb, gt_rgb)
@@ -915,7 +949,7 @@ class SplatfactoModel(Model):
batch: ground truth batch corresponding to outputs
metrics_dict: dictionary of metrics, some of which we can use for loss
"""
gt_img = self.get_gt_img(batch["image"])
gt_img = self.composite_with_background(self.get_gt_img(batch["image"]), outputs["background"])
pred_img = outputs["rgb"]
# Set masked part of both ground-truth and rendered image to black.
@@ -974,7 +1008,7 @@ class SplatfactoModel(Model):
Returns:
A dictionary of metrics.
"""
gt_rgb = self.get_gt_img(batch["image"])
gt_rgb = self.composite_with_background(self.get_gt_img(batch["image"]), outputs["background"])
d = self._get_downscale_factor()
if d > 1:
# torchvision can be slow to import, so we do it lazily.
@@ -16,6 +16,7 @@
import shutil
from dataclasses import dataclass
from typing import Literal
from nerfstudio.process_data import equirect_utils, process_data_utils
from nerfstudio.process_data.colmap_converter_to_nerfstudio_dataset import ColmapConverterToNerfstudioDataset
@@ -36,6 +37,10 @@ class VideoToNerfstudioDataset(ColmapConverterToNerfstudioDataset):
"""Target number of frames to use per video, results may not be exact."""
percent_radius_crop: float = 1.0
"""Create circle crop mask. The radius is the percent of the image diagonal."""
matching_method: Literal["exhaustive", "sequential", "vocab_tree"] = "sequential"
"""Feature matching method to use. Vocab tree is recommended for a balance of speed
and accuracy. Exhaustive is slower but more accurate. Sequential is faster but
should only be used for videos."""
def main(self) -> None:
"""Process video into a nerfstudio dataset."""
+1 -1
View File
@@ -63,7 +63,7 @@ dependencies = [
"xatlas",
"trimesh>=3.20.2",
"timm==0.6.7",
"gsplat==0.1.3",
"gsplat>=0.1.5",
"pytorch-msssim",
"pathos",
"packaging"