mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-20 22:56:03 +08:00
Make DensityGrid plug-and-play for the base Graph and the base Sampler. (#205)
This commit is contained in:
1 parent
512842938a
commit
31c2c2608d
11 files changed
+327
-99
No files matched your search
@@ -22,6 +22,7 @@ from torch import nn
|
||||
from torchtyping import TensorType
|
||||
|
||||
import nerfactory.cuda as nerfactory_cuda
|
||||
from nerfactory.cameras.rays import RaySamples
|
||||
|
||||
|
||||
def _create_grid_coords(resolution: int, device: torch.device = "cpu") -> TensorType["n_coords", 3]:
|
||||
@@ -44,9 +45,25 @@ def _create_grid_coords(resolution: int, device: torch.device = "cpu") -> Tensor
|
||||
class DensityGrid(nn.Module):
|
||||
"""Cascaded multi-res density grids.
|
||||
|
||||
The multi-res grids are all centerd at [center, center, center] in the world space.
|
||||
The first grid covers regions (center - base_scale / 2, center + base_scale / 2)
|
||||
and the followings grows by 2x scale each.
|
||||
The DensityGrid contains multi-level (`num_cascades`) grids from finest (0) to the coarsest (num_cascades-1).
|
||||
All levels of grid have the same `resolution` but cover different regions in the space. The finest grid is the
|
||||
smallest one, which has scale of `base_scale` and centered at `center`. Every next level of grid grows by 2x
|
||||
on the scale but all shares the same center.
|
||||
|
||||
For example:
|
||||
- `center=0`, `base_scale=3`, `num_cascades=1` would create one grid (lvl=0) covers [-1.5, -1.5, -1.5, 1.5, 1.5, 1.5].
|
||||
- `center=0`, `base_scale=3`, `num_cascades=2` would create one grid (lvl=0) covers [-1.5, -1.5, -1.5, 1.5, 1.5, 1.5],
|
||||
and one more grid (lvl=1) covers [-3, -3, -3, 3, 3, 3].
|
||||
|
||||
Normally you can simply set the `center` to the scene center, and `base_scale` to be the scene scale, and use
|
||||
`num_cascades=1` to create a single level density grid that covers the scene.
|
||||
|
||||
If, for example, you cares more about the central region of the scene and want to have more fine-grained density
|
||||
grid at central region, you can set the `base_scale` to be the scale of the central region that you care. And adjust
|
||||
`num_cascades` so that the coarsest level grid covers your entire scene. You should follow this rule so that the
|
||||
multi-level grids can cover your scene:
|
||||
|
||||
scene_scale <= base_scale * 2 ** (num_cascades - 1)
|
||||
|
||||
TODO(ruilongli): support `center` and `base_scale` with 3-dim.
|
||||
|
||||
@@ -86,6 +103,9 @@ class DensityGrid(nn.Module):
|
||||
self.register_buffer("grid_coords", grid_coords)
|
||||
self.register_buffer("grid_indices", grid_indices)
|
||||
|
||||
mean_density = torch.zeros((1,))
|
||||
self.register_buffer("mean_density", mean_density)
|
||||
|
||||
@torch.no_grad()
|
||||
def get_all_cells(self) -> List[Tuple[TensorType["n_cells"], TensorType["n_cells", 3]]]:
|
||||
"""Returns all cells of the grid.
|
||||
@@ -171,23 +191,39 @@ class DensityGrid(nn.Module):
|
||||
# ema update
|
||||
valid_mask = (self.density_grid >= 0) & (tmp_grid >= 0)
|
||||
self.density_grid[valid_mask] = torch.maximum(self.density_grid[valid_mask] * decay, tmp_grid[valid_mask])
|
||||
mean_density = self.density_grid.mean().item()
|
||||
self.mean_density.data = self.density_grid.mean()
|
||||
|
||||
# pack to bitfield
|
||||
self.density_bitfield.data = nerfactory_cuda.packbits(self.density_grid, min(mean_density, density_threshold))
|
||||
self.density_bitfield.data = nerfactory_cuda.packbits(
|
||||
self.density_grid, min(self.mean_density.item(), density_threshold)
|
||||
)
|
||||
|
||||
# TODO(ruilongli): max pooling? https://github.com/NVlabs/instant-ngp/blob/master/src/testbed_nerf.cu#L578
|
||||
|
||||
@torch.no_grad()
|
||||
def get_densities(self, xyzs: TensorType[..., 3]) -> TensorType[..., 1]:
|
||||
"""Trilinear interpolation to get the density values.
|
||||
def get_densities(self, ray_samples: RaySamples) -> TensorType[..., 1]:
|
||||
"""Get the density values stored in the density grid.
|
||||
Args:
|
||||
xyzs (TensorType[..., 3]): 3D querry coordinate
|
||||
ray_samples RaySamples: ray samples
|
||||
Returns:
|
||||
TensorType[..., 1]: Density values
|
||||
"""
|
||||
# TODO(ruilongli) pass in RaySamples because we need dt to decide mip level.
|
||||
raise NotImplementedError("We haven't implement the logic of querying densities from cascaded grid.")
|
||||
positions = ray_samples.frustums.get_positions()
|
||||
deltas = ray_samples.frustums.ends - ray_samples.frustums.starts
|
||||
|
||||
_mip_levels, indices, _occupancies = nerfactory_cuda.occupancy_query(
|
||||
positions.reshape(-1, 3),
|
||||
deltas.reshape(-1),
|
||||
self.center,
|
||||
self.base_scale,
|
||||
self.num_cascades,
|
||||
self.resolution,
|
||||
self.density_bitfield,
|
||||
)
|
||||
|
||||
densities = self.density_grid.reshape(-1)[indices.long()]
|
||||
densities = densities.view(*positions.shape[:-1], 1)
|
||||
return densities
|
||||
|
||||
def forward(self, x):
|
||||
"""Not implemented."""
|
||||
|
||||
Reference in new issue
Block a user