Update Instant-NGP & Density Grid with CUDA supports. Much faster and better PSNR. (#193)

This commit is contained in:
Ruilong Li(李瑞龙) authored and GitHub committed 2022-07-24 22:22:44 -07:00
1 parent 31b52ae186
commit 12cd321e1a
39 files changed
+1721 -650

No files matched your search

+3 -3
View File
@@ -19,12 +19,12 @@ Base class for the graphs.
from abc import abstractmethod
from typing import Dict, Optional, Tuple
from torch import nn
import torch
from torch import nn
from torchtyping import TensorType
from pyrad.fields.modules.field_heads import FieldHeadNames
from pyrad.cameras.rays import Frustums, RaySamples
from pyrad.fields.modules.field_heads import FieldHeadNames
from pyrad.utils.misc import is_not_none
@@ -32,7 +32,7 @@ class Field(nn.Module):
"""Base class for fields."""
def density_fn(self, positions):
"""Returns only the density. Used primarily with the occupancy grid."""
"""Returns only the density. Used primarily with the density grid."""
# Need to figure out a better way to descibe positions with a ray.
ray_samples = RaySamples(
frustums=Frustums(
File renamed without changes.
+194
View File
@@ -0,0 +1,194 @@
# Copyright 2022 The Plenoptix Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Code to implement the density grid.
"""
from typing import Callable, List, NoReturn, Tuple
import torch
from torch import nn
from torchtyping import TensorType
import pyrad.cuda as pyrad_cuda
def _create_grid_coords(resolution: int, device: torch.device = "cpu") -> TensorType["n_coords", 3]:
"""Create 3D grid coordinates
Args:
resolution (int): The 3D resolution of the grid.
device (torch.device): Which device you want the returned tensor to live int.
Returns:
TensorType["n_coords", 3]: All grid coordinates with shape [res * res * res, 3]
"""
arrange = torch.arange(resolution, device=device)
grid_x, grid_y, grid_z = torch.meshgrid([arrange, arrange, arrange], indexing="ij")
coords = torch.stack([grid_x, grid_y, grid_z]) # [3, res, res, res]
coords = coords.reshape(3, -1).t().contiguous() # [N, 3]
return coords
class DensityGrid(nn.Module):
"""Cascaded multi-res density grids.
The multi-res grids are all centerd at [center, center, center] in the world space.
The first grid covers regions (center - base_scale / 2, center + base_scale / 2)
and the followings grows by 2x scale each.
TODO(ruilongli): support `center` and `base_scale` with 3-dim.
Args:
center (float, optional): Center of all the grids in the world space. Defaults to 0.
base_scale (float, optional): Center of the scale of the base level grid. Defaults to 1.
num_cascades (int, optional): Number of the cascaded multi-res levels. Defaults to 1.
resolution (int, optional): Resolution of the grid. Defaults to 128.
update_every_num_iters (int, optional): How frequently to update the grid values. Defaults to 16.
"""
def __init__(
self,
center: float = 0.0,
base_scale: float = 1.0,
num_cascades: int = 1,
resolution: int = 128,
update_every_num_iters: int = 16,
) -> None:
super().__init__()
self.center = center
self.base_scale = base_scale
self.num_cascades = num_cascades # the number of levels (i.e, cascades)
self.resolution = resolution
self.update_every_num_iters = update_every_num_iters
density_grid = torch.zeros([self.num_cascades] + [self.resolution**3])
self.register_buffer("density_grid", density_grid)
density_bitfield = torch.zeros([self.num_cascades * self.resolution**3 // 8], dtype=torch.uint8)
self.register_buffer("density_bitfield", density_bitfield)
# Integer grid coords / indices that do not related to cascades
grid_coords = _create_grid_coords(resolution)
# TODO(ruilongli): hacky way for now. support cpu version of morton3D?
grid_indices = pyrad_cuda.morton3D(grid_coords.to("cuda:0")).to("cpu")
self.register_buffer("grid_coords", grid_coords)
self.register_buffer("grid_indices", grid_indices)
@torch.no_grad()
def get_all_cells(self) -> List[Tuple[TensorType["n_cells"], TensorType["n_cells", 3]]]:
"""Returns all cells of the grid.
Returns:
List[Tuple[TensorType["n_cells"], TensorType["n_cells", 3]]]: All grid indices and grid
coordinates of all cascade levels.
"""
return [(self.grid_indices, self.grid_coords)] * self.num_cascades
@torch.no_grad()
def sample_uniform_and_occupied_cells(self, n: int) -> List[Tuple[TensorType["n"], TensorType["n", 3]]]:
"""Samples both n uniform and occupied cells (per cascade)
Args:
n: Number of cells to be sampled.
Returns:
List[Tuple[TensorType["n"], TensorType["n", 3]]]: Sampled grid indices and grid
coordinates of all cascade levels.
"""
device = self.density_grid.device
cells = []
for c in range(self.num_cascades):
uniform_coords = torch.randint(self.resolution, (n, 3), device=device)
uniform_indices = pyrad_cuda.morton3D(uniform_coords)
occupied_indices = torch.nonzero(self.density_grid[c] > 0)[:, 0]
if n < len(occupied_indices):
selector = torch.randint(len(occupied_indices), (n,), device=device)
occupied_indices = occupied_indices[selector]
occupied_coords = pyrad_cuda.morton3D_invert(occupied_indices)
# concatenate
coords = torch.cat([uniform_coords, occupied_coords], dim=0)
indices = torch.cat([uniform_indices, occupied_indices], dim=0)
cells.append((indices, coords))
return cells
@torch.no_grad()
def update_density_grid(
self,
density_eval_func: Callable,
step: int,
density_threshold: float = 2, # 0.01 / (SQRT3 / 1024 * 3)
decay: float = 0.95,
) -> NoReturn:
"""Update the density grid in EMA way.
Args:
density_eval_func (Callable): A Callable function that takes in sample positions (N, 3) and
returns densities (N, 1).
step (int): The current training step. Instant-NGP has a warmup stage where all cells are updated.
After certain amount of steps (256), it speeds up by only update sampled cells.
density_threshold (float): The threshold to prune cells. Recommand to calculate it using this rule:
`weight_threshold / dt`. For example for Instant-NGP on Lego, the minimum step size `dt` is
`3` (scale of the scene) * `sqrt(3)` / `1024` (number of samples). And if we want to the weight
threshold to be `0.01`, we will get `0.01 / (3 * sqrt(3) / 1024) ~= 2.` for density threshold.
decay (float): EMA decay for density updating.
"""
# create temporary grid
tmp_grid = -torch.ones_like(self.density_grid)
if step < 256:
cells = self.get_all_cells()
else:
N = self.resolution**3 // 4
cells = self.sample_uniform_and_occupied_cells(N)
# infer sigmas
for mip_level in range(self.num_cascades):
mip_scale = 2 ** (-mip_level) / self.base_scale
indices, coords = cells[mip_level]
# `coords` denotes the i-th cell. It's a poor naming here.
# the actually coordinates in world space x has mapping to
# the cell like this:
# x \in [0, 1/res] maps to 0-th cell.
# x \in [1 - 1/res, 1] maps to (res-1)-th cell.
x = (coords + torch.rand_like(coords.float())) / self.resolution
x = (x - 0.5) / mip_scale + self.center
tmp_grid[mip_level, indices] = density_eval_func(x).squeeze(-1)
# ema update
valid_mask = (self.density_grid >= 0) & (tmp_grid >= 0)
self.density_grid[valid_mask] = torch.maximum(self.density_grid[valid_mask] * decay, tmp_grid[valid_mask])
mean_density = self.density_grid.mean().item()
# pack to bitfield
self.density_bitfield.data = pyrad_cuda.packbits(self.density_grid, min(mean_density, density_threshold))
# TODO(ruilongli): max pooling? https://github.com/NVlabs/instant-ngp/blob/master/src/testbed_nerf.cu#L578
@torch.no_grad()
def get_densities(self, xyzs: TensorType[..., 3]) -> TensorType[..., 1]:
"""Trilinear interpolation to get the density values.
Args:
xyzs (TensorType[..., 3]): 3D querry coordinate
Returns:
TensorType[..., 1]: Density values
"""
# TODO(ruilongli) pass in RaySamples because we need dt to decide mip level.
raise NotImplementedError("We haven't implement the logic of querying densities from cascaded grid.")
def forward(self, x):
"""Not implemented."""
raise RuntimeError("Shouldn't be called!")
+4 -4
View File
@@ -22,11 +22,11 @@ from typing import Tuple
import torch
from torch.nn.parameter import Parameter
from pyrad.cameras.rays import RaySamples
from pyrad.fields.base import Field
from pyrad.fields.modules.encoding import Encoding, HashEncoding, SHEncoding
from pyrad.fields.modules.field_heads import FieldHeadNames
from pyrad.fields.base import Field
from pyrad.fields.nerf_field import NeRFField
from pyrad.cameras.rays import RaySamples
from pyrad.utils.activations import trunc_exp
try:
@@ -113,7 +113,7 @@ class TCNNInstantNGPField(Field):
# Rectifying the density with an exponential is much more stable than a ReLU or
# softplus, because it enables high post-activation (float32) density outputs
# from smaller internal (float16) parameters.
density = trunc_exp(density_before_activation)
density = trunc_exp(density_before_activation.to(positions))
return density, base_mlp_out
def get_outputs(self, ray_samples: RaySamples, density_embedding=None):
@@ -123,7 +123,7 @@ class TCNNInstantNGPField(Field):
directions_flat = directions.view(-1, 3)
d = self.direction_encoding(directions_flat)
h = torch.cat([d, density_embedding.view(-1, self.geo_feat_dim)], dim=-1)
rgb = self.mlp_head(h).view(*ray_samples.frustums.directions.shape[:-1], -1)
rgb = self.mlp_head(h).view(*ray_samples.frustums.directions.shape[:-1], -1).to(directions)
return {FieldHeadNames.RGB: rgb}
@@ -1,144 +0,0 @@
# Copyright 2022 The Plenoptix Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Code to implement the density grid.
"""
import torch
import torch.nn.functional as F
from torch import nn
from torchtyping import TensorType
from pyrad.utils.misc import is_not_none
class OccupancyGrid(nn.Module):
"""Module to keep track of the density and occupancy."""
def __init__(
self, num_cascades: int = 1, resolution: int = 128, aabb=None, update_every_num_iters: int = 16
) -> None:
super().__init__()
assert is_not_none(aabb), "The axis-aligned bounding box aabb is not defined!"
self.num_cascades = num_cascades # the number of levels (i.e, cascades)
self.resolution = resolution
self.register_buffer("aabb", aabb) # axis-aligned bounding box
occupancy_grid = torch.ones(
[self.num_cascades] + [self.resolution] * 3
) # (num_cascades, x_resolution, y_resolution, z_resolution)
# not a module parameter, but it still should be saved and part of `state_dict`
self.register_buffer("occupancy_grid", occupancy_grid)
self.mean_density = 0.0
self.update_every_num_iters = update_every_num_iters
def reset(self):
"""Zero out the occupancy grid."""
self.occupancy_grid.zero_()
@torch.no_grad()
def mark_untrained_grid(self):
"""Marks the locations in the density grid that are never seen by training views."""
# TODO: only query points that are covered by the training cameras viewing frustrum
# We'll need to pass in the camera poses and intrinsics to this function
raise NotImplementedError()
@torch.no_grad()
def update_occupancy_grid(self, density_fn, decay=0.95, split_size=64):
"""Update the occupancy grid with density values.
Args:
density_fn: Function from takes points in the shape (..., 3) and
returns a tensor of densities of shape (..., 1).
"""
# TODO(ethan): remove this pylint disable
# pylint: disable=invalid-name
device = self.occupancy_grid.device
assert self.num_cascades == 1, "Still need to implement code for multiple cascades."
temp_occupancy_grid = torch.zeros_like(self.occupancy_grid)
X_length, Y_length, Z_length = self.aabb[1] - self.aabb[0]
X_half_voxel_length = 0.5 * X_length / self.resolution
Y_half_voxel_length = 0.5 * Y_length / self.resolution
Z_half_voxel_length = 0.5 * Z_length / self.resolution
half_voxel_lengths = torch.tensor(
[X_half_voxel_length, Y_half_voxel_length, Z_half_voxel_length], device=device
)
X = torch.linspace(
self.aabb[0][0] + X_half_voxel_length, self.aabb[1][0] - X_half_voxel_length, self.resolution, device=device
).split(split_size)
Y = torch.linspace(
self.aabb[0][1] + Y_half_voxel_length, self.aabb[1][1] - Y_half_voxel_length, self.resolution, device=device
).split(split_size)
Z = torch.linspace(
self.aabb[0][2] + Z_half_voxel_length, self.aabb[1][2] - Z_half_voxel_length, self.resolution, device=device
).split(split_size)
for x_index, xs in enumerate(X):
for y_index, ys in enumerate(Y):
for z_index, zs in enumerate(Z):
x_len, y_len, z_len = len(xs), len(ys), len(zs)
grid_x, grid_y, grid_z = torch.meshgrid(xs, ys, zs, indexing="ij")
xyzs = torch.stack([grid_x, grid_y, grid_z], dim=-1) # (resolution, resolution,, resolution, 3)
# add noise to the points
xyzs_rand = (
torch.rand_like(xyzs) * half_voxel_lengths
) # range [-half_voxel_length, half_voxel_length] for each dimension
xyzs += xyzs_rand
densities = density_fn(xyzs)
assert densities.shape[-1] == 1
temp_occupancy_grid[
0,
x_index * split_size : x_index * split_size + x_len,
y_index * split_size : y_index * split_size + y_len,
z_index * split_size : z_index * split_size + z_len,
] = densities[..., 0]
# exponential moving average update
valid_mask = self.occupancy_grid >= 0.0
self.occupancy_grid[valid_mask] = torch.maximum(
self.occupancy_grid[valid_mask] * decay, temp_occupancy_grid[valid_mask]
)
self.mean_density = torch.mean(self.occupancy_grid[valid_mask]).item()
@torch.no_grad()
def get_densities(self, xyzs: TensorType[..., 3]) -> TensorType[..., 1]:
"""Trilinear interpolation to get the density values.
Args:
xyzs (TensorType[..., 3]): 3D querry coordinate
Returns:
TensorType[..., 1]: Density values
"""
occupancy_grid = self.occupancy_grid[None, ...] # shape (1, num_cascades, X_res, Y_res, Z_res)
xyzs_shape = xyzs.shape
xyzs_reshaped = xyzs.view(1, -1, 1, 1, 3)
voxel_lengths = self.aabb[1] - self.aabb[0]
xyzs_reshaped_normalized = ((xyzs_reshaped - self.aabb[0]) / voxel_lengths) * 2.0 - 1.0
densities = F.grid_sample(
occupancy_grid.permute(
0, 1, 4, 3, 2
), # (xyz to zyx) for grid sample useage because the xyzs have xyz ordering
xyzs_reshaped_normalized,
align_corners=True,
padding_mode="zeros",
)
densities = densities.view(*xyzs_shape[:-1], 1)
return densities
def forward(self, x):
"""Not implemented."""
raise NotImplementedError()