diff --git a/configs/graph_default.yaml b/configs/graph_default.yaml index 2345b923e..28c26f7e2 100644 --- a/configs/graph_default.yaml +++ b/configs/graph_default.yaml @@ -90,6 +90,15 @@ graph: loss_coefficients: rgb_loss_coarse: 1.0 rgb_loss_fine: 1.0 + enable_density_field: false + density_field_config: + # Please see the docstr for DensityGrid on how to set those arguments. + _target_: nerfactory.fields.density_fields.density_grid.DensityGrid + center: 0.0 # simply set it as the center of the scene bbox + base_scale: 3.0 # simply set it as the scale of the scene bbox + num_cascades: 1 # if using more than 1 cascade, the `base_scale` can be smaller than scene scale. + resolution: 128 + update_every_num_iters: 16 optimizers: fields: diff --git a/nerfactory/cuda/__init__.py b/nerfactory/cuda/__init__.py index 30d0a4b7c..e6347705d 100644 --- a/nerfactory/cuda/__init__.py +++ b/nerfactory/cuda/__init__.py @@ -25,6 +25,7 @@ morton3D_invert = _C.morton3D_invert raymarching = _C.raymarching volumetric_rendering_forward = _C.volumetric_rendering_forward volumetric_rendering_backward = _C.volumetric_rendering_backward +occupancy_query = _C.occupancy_query # pylint: disable=abstract-method,arguments-differ class VolumeRenderer(torch.autograd.Function): @@ -77,3 +78,18 @@ class VolumeRenderer(torch.autograd.Function): ) # corresponds to the input argument list of forward() return None, None, None, grad_sigmas, grad_rgbs + + +def unpackbits(x: torch.Tensor) -> torch.Tensor: + """Unpack uint8 bits back to a boolen mask. + + Args: + x: uint8 bit tensor with shape [N] + + Returns: + unpacked boolen tensor with shape [N * 8] + """ + assert x.dtype == torch.uint8 and x.dim() == 1 + bits = x.element_size() * 8 + mask = 2 ** torch.arange(bits - 1, -1, -1).to(x) + return x.unsqueeze(-1).bitwise_and(mask).ne(0).flip(-1).bool().reshape(-1) diff --git a/nerfactory/cuda/csrc/pyrad.cpp b/nerfactory/cuda/csrc/nerfactory.cpp similarity index 84% rename from nerfactory/cuda/csrc/pyrad.cpp rename to nerfactory/cuda/csrc/nerfactory.cpp index e16920fc3..569c47306 100644 --- a/nerfactory/cuda/csrc/pyrad.cpp +++ b/nerfactory/cuda/csrc/nerfactory.cpp @@ -56,6 +56,18 @@ std::vector volumetric_rendering_backward( torch::Tensor rgbs ) {}; +std::vector occupancy_query( + // samples + const torch::Tensor positions, + const torch::Tensor deltas, + // density grid + const float grid_center, + const float grid_scale, + const int grid_cascades, + const int grid_size, + const torch::Tensor grid_bitfield +) {}; + PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { m.def("packbits", &packbits); @@ -65,4 +77,5 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) m.def("raymarching", &raymarching); m.def("volumetric_rendering_forward", &volumetric_rendering_forward); m.def("volumetric_rendering_backward", &volumetric_rendering_backward); + m.def("occupancy_query", &occupancy_query); } \ No newline at end of file diff --git a/nerfactory/cuda/csrc/pyrad.cu b/nerfactory/cuda/csrc/nerfactory.cu similarity index 84% rename from nerfactory/cuda/csrc/pyrad.cu rename to nerfactory/cuda/csrc/nerfactory.cu index 1a72b3d8a..6142ceaae 100644 --- a/nerfactory/cuda/csrc/pyrad.cu +++ b/nerfactory/cuda/csrc/nerfactory.cu @@ -56,6 +56,18 @@ std::vector volumetric_rendering_backward( torch::Tensor rgbs ); +std::vector occupancy_query( + // samples + const torch::Tensor positions, + const torch::Tensor deltas, + // density grid + const float grid_center, + const float grid_scale, + const int grid_cascades, + const int grid_size, + const torch::Tensor grid_bitfield +); + PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { m.def("packbits", &packbits); @@ -65,4 +77,5 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) m.def("raymarching", &raymarching); m.def("volumetric_rendering_forward", &volumetric_rendering_forward); m.def("volumetric_rendering_backward", &volumetric_rendering_backward); + m.def("occupancy_query", &occupancy_query); } \ No newline at end of file diff --git a/nerfactory/cuda/csrc/raymarching.cu b/nerfactory/cuda/csrc/raymarching.cu index 530e0dd1a..d6c53629b 100644 --- a/nerfactory/cuda/csrc/raymarching.cu +++ b/nerfactory/cuda/csrc/raymarching.cu @@ -15,44 +15,45 @@ inline __device__ float max_step_size(uint32_t grid_cascades, uint32_t grid_size // stepping in larger scenes. inline __device__ float calc_dt(float t, float cone_angle, float dt_min, float dt_max) { // TODO(ruilongli): scene_scale related to cone_angle? - return __clamp(t * cone_angle, dt_min, dt_max); + return __clamp(t * cone_angle, dt_min, dt_max); } inline __device__ int mip_from_pos(float x, float y, float z, uint32_t grid_cascades, float grid_center, float grid_scale) { float maxval = fmaxf(fmaxf(fabsf(x - grid_center), fabsf(y - grid_center)), fabsf(z - grid_center)) / grid_scale; - int exponent; frexpf(maxval, &exponent); - return min(grid_cascades-1, max(0, exponent+1)); + int exponent; frexpf(maxval, &exponent); + return min(grid_cascades-1, max(0, exponent+1)); } inline __device__ int mip_from_dt( float x, float y, float z, uint32_t grid_cascades, float dt, int grid_size, float grid_center, float grid_scale ) { - int mip = mip_from_pos(x, y, z, grid_cascades, grid_center, grid_scale); - dt *= 2 * grid_size; - if (dt<1.f) return mip; // exponent would be zero - int exponent; frexpf(dt, &exponent); - return min(grid_cascades-1, max(exponent, mip)); + int mip = mip_from_pos(x, y, z, grid_cascades, grid_center, grid_scale); + dt *= 2 * grid_size; + if (dt<1.f) return mip; // exponent would be zero + int exponent; frexpf(dt, &exponent); + return min(grid_cascades-1, max(exponent, mip)); } inline __device__ uint32_t grid_mip_offset(uint32_t mip, int grid_size) { - return (grid_size * grid_size * grid_size) * mip; + return (grid_size * grid_size * grid_size) * mip; } inline __device__ uint32_t cascaded_grid_idx_at( float x, float y, float z, uint32_t mip, int grid_size, float grid_center, float grid_scale ) { - float mip_scale = scalbnf(1.0f, -mip) / grid_scale; + // TODO(ruilongli): if the x, y, z is outside the aabb, it will be clipped into aabb!!! We should just return false + float mip_scale = scalbnf(1.0f, -mip) / grid_scale; int ix = (int)((mip_scale * (x - grid_center) + 0.5f) * grid_size); int iy = (int)((mip_scale * (y - grid_center) + 0.5f) * grid_size); int iz = (int)((mip_scale * (z - grid_center) + 0.5f) * grid_size); // printf("[input] x %f, y %f, z %f, mip %d, grid_size %d, grid_center %f\n", x, y, z, mip, grid_size, grid_center); // printf("[output] mip_scale %f, ix %d iy %d, iz %d\n", mip_scale, ix, iy, iz); - uint32_t idx = __morton3D( - __clamp(ix, 0, grid_size-1), - __clamp(iy, 0, grid_size-1), - __clamp(iz, 0, grid_size-1) - ); - return idx; + uint32_t idx = __morton3D( + __clamp(ix, 0, grid_size-1), + __clamp(iy, 0, grid_size-1), + __clamp(iz, 0, grid_size-1) + ); + return idx; } inline __device__ bool grid_occupied_at( @@ -60,11 +61,11 @@ inline __device__ bool grid_occupied_at( const uint8_t* grid_bitfield, uint32_t mip, int grid_size, float grid_center, float grid_scale ) { - uint32_t idx = ( + uint32_t idx = ( cascaded_grid_idx_at(x, y, z, mip, grid_size, grid_center, grid_scale) + grid_mip_offset(mip, grid_size) ); - return grid_bitfield[idx/8] & (1<<(idx%8)); + return grid_bitfield[idx/8] & (1<<(idx%8)); } inline __device__ float distance_to_next_voxel( @@ -74,13 +75,13 @@ inline __device__ float distance_to_next_voxel( float res ) { // dda like step // TODO: warning: expression has no effect? - x, y, z = res * x, res * y, res * z; - float tx = (floorf(x + 0.5f + 0.5f * __sign(dir_x)) - x) * idir_x; - float ty = (floorf(y + 0.5f + 0.5f * __sign(dir_y)) - y) * idir_y; - float tz = (floorf(z + 0.5f + 0.5f * __sign(dir_z)) - z) * idir_z; - float t = min(min(tx, ty), tz); + x, y, z = res * x, res * y, res * z; + float tx = (floorf(x + 0.5f + 0.5f * __sign(dir_x)) - x) * idir_x; + float ty = (floorf(y + 0.5f + 0.5f * __sign(dir_y)) - y) * idir_y; + float tz = (floorf(z + 0.5f + 0.5f * __sign(dir_z)) - z) * idir_z; + float t = min(min(tx, ty), tz); - return fmaxf(t / res, 0.0f); + return fmaxf(t / res, 0.0f); } inline __device__ float advance_to_next_voxel( @@ -89,14 +90,14 @@ inline __device__ float advance_to_next_voxel( float dir_x, float dir_y, float dir_z, float idir_x, float idir_y, float idir_z, float res, float dt_min) { - // Regular stepping (may be slower but matches non-empty space) - float t_target = t + distance_to_next_voxel( + // Regular stepping (may be slower but matches non-empty space) + float t_target = t + distance_to_next_voxel( x, y, z, dir_x, dir_y, dir_z, idir_x, idir_y, idir_z, res ); - do { - t += dt_min; - } while (t < t_target); - return t; + do { + t += dt_min; + } while (t < t_target); + return t; } template @@ -143,10 +144,10 @@ __global__ void kernel_raymarching( // TODO(ruilongli): pre-compute `dt_min`, `dt_max` as it is a constant? float dt_min = min_step_size(num_steps) * step_scale; float dt_max = max_step_size(grid_cascades, grid_size) * grid_scale; - + // first pass to compute an accurate number of steps - uint32_t j = 0; - float t0 = near; + uint32_t j = 0; + float t0 = near; float dt = calc_dt(t0, cone_angle, dt_min, dt_max); float t1 = t0 + dt; float t_mid = (t0 + t1) * 0.5f; @@ -156,33 +157,33 @@ __global__ void kernel_raymarching( const float x = ox + t_mid * dx; const float y = oy + t_mid * dy; const float z = oz + t_mid * dz; - uint32_t mip = mip_from_dt(x, y, z, grid_cascades, dt, grid_size, grid_center, grid_scale); + uint32_t mip = mip_from_dt(x, y, z, grid_cascades, dt, grid_size, grid_center, grid_scale); if (grid_occupied_at(x, y, z, grid_bitfield, mip, grid_size, grid_center, grid_scale)) { ++j; // march to next sample - t0 = t1; + t0 = t1; dt = calc_dt(t0, cone_angle, dt_min, dt_max); t1 = t0 + dt; t_mid = (t0 + t1) * 0.5f; - } + } else { // march to next sample - float res = (grid_size >> mip) * grid_scale; - t_mid = advance_to_next_voxel( + float res = (grid_size >> mip) * grid_scale; + t_mid = advance_to_next_voxel( t_mid, x, y, z, dx, dy, dz, rdx, rdy, rdz, res, dt_min ); dt = calc_dt(t_mid, cone_angle, dt_min, dt_max); t0 = t_mid - dt * 0.5f; t1 = t_mid + dt * 0.5f; - } - } + } + } if (j == 0) return; uint32_t numsteps = j; - uint32_t base = atomicAdd(numsteps_counter, numsteps); + uint32_t base = atomicAdd(numsteps_counter, numsteps); if (base + numsteps > max_total_samples) return; - + // locate origins_out += base * 3; dirs_out += base * 3; @@ -191,13 +192,13 @@ __global__ void kernel_raymarching( uint32_t ray_idx = atomicAdd(rays_counter, 1); - packed_info_out[ray_idx * 3 + 0] = i; // ray idx in {rays_o, rays_d} + packed_info_out[ray_idx * 3 + 0] = i; // ray idx in {rays_o, rays_d} packed_info_out[ray_idx * 3 + 1] = base; // point idx start. packed_info_out[ray_idx * 3 + 2] = numsteps; // point idx shift. // Second round j = 0; - t0 = near; + t0 = near; dt = calc_dt(t0, cone_angle, dt_min, dt_max); t1 = t0 + dt; t_mid = (t0 + t1) / 2.; @@ -207,7 +208,7 @@ __global__ void kernel_raymarching( const float x = ox + t_mid * dx; const float y = oy + t_mid * dy; const float z = oz + t_mid * dz; - uint32_t mip = mip_from_dt(x, y, z, grid_cascades, dt, grid_size, grid_center, grid_scale); + uint32_t mip = mip_from_dt(x, y, z, grid_cascades, dt, grid_size, grid_center, grid_scale); if (grid_occupied_at(x, y, z, grid_bitfield, mip, grid_size, grid_center, grid_scale)) { origins_out[j * 3 + 0] = ox; @@ -220,25 +221,75 @@ __global__ void kernel_raymarching( ends_out[j] = t1; ++j; // march to next sample - t0 = t1; + t0 = t1; dt = calc_dt(t0, cone_angle, dt_min, dt_max); t1 = t0 + dt; t_mid = (t0 + t1) * 0.5f; - } + } else { // march to next sample - float res = (grid_size >> mip) * grid_scale; - t_mid = advance_to_next_voxel( + float res = (grid_size >> mip) * grid_scale; + t_mid = advance_to_next_voxel( t_mid, x, y, z, dx, dy, dz, rdx, rdy, rdz, res, dt_min ); dt = calc_dt(t_mid, cone_angle, dt_min, dt_max); t0 = t_mid - dt * 0.5f; t1 = t_mid + dt * 0.5f; - } - } + } + } return; } + +template +__global__ void kernel_occupancy_query( + // samples info + const uint32_t n_samples, + const scalar_t* positions, + const scalar_t* deltas, + // density grid + const float grid_center, + const float grid_scale, + const int grid_cascades, + const int grid_size, + const uint8_t* grid_bitfield, + // outputs + int32_t* mip_levels, // output mip level + int32_t* indices, // output indices + bool* occupancies // output occupancy +) { + CUDA_GET_THREAD_ID(i, n_samples); + + // locate + positions += i * 3; + deltas += i; + mip_levels += i; + indices += i; + occupancies += i; + + // current point + const float x = positions[0]; + const float y = positions[1]; + const float z = positions[2]; + const float dt = deltas[0]; + + int32_t mip = mip_from_dt(x, y, z, grid_cascades, dt, grid_size, grid_center, grid_scale); + mip_levels[0] = mip; + + int32_t idx = ( + cascaded_grid_idx_at(x, y, z, mip, grid_size, grid_center, grid_scale) + + grid_mip_offset(mip, grid_size) + ); + indices[0] = idx; + + if (grid_bitfield) { + bool occ = grid_bitfield[idx/8] & (1<<(idx%8)); + occupancies[0] = occ; + } + return; +} + + /** * @brief Sample points by ray marching. * @@ -344,4 +395,58 @@ std::vector raymarching( })); return {packed_info, origins, dirs, starts, ends}; -} \ No newline at end of file +} + + +std::vector occupancy_query( + // samples + const torch::Tensor positions, + const torch::Tensor deltas, + // density grid + const float grid_center, + const float grid_scale, + const int grid_cascades, + const int grid_size, + const torch::Tensor grid_bitfield +) { + DEVICE_GUARD(positions); + + CHECK_INPUT(positions); + CHECK_INPUT(deltas); + CHECK_INPUT(grid_bitfield); + + const int n_samples = positions.size(0); + + const int threads = 256; + const int blocks = CUDA_N_BLOCKS_NEEDED(n_samples, threads); + + // outputs + torch::Tensor mip_levels = torch::empty({n_samples}, positions.options().dtype(torch::kInt32)); + torch::Tensor indices = torch::empty({n_samples}, positions.options().dtype(torch::kInt32)); + torch::Tensor occupancies = torch::empty({n_samples}, positions.options().dtype(torch::kBool)); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + positions.scalar_type(), + "occupancy_query", + ([&] + { kernel_occupancy_query<<>>( + // samples + n_samples, + positions.data_ptr(), + deltas.data_ptr(), + // density grid + grid_center, + grid_scale, + grid_cascades, + grid_size, + grid_bitfield.data_ptr(), + // outputs + mip_levels.data_ptr(), + indices.data_ptr(), + occupancies.data_ptr() + ); + })); + + return {mip_levels, indices, occupancies}; +} + diff --git a/nerfactory/cuda/csrc/volume_rendering.cu b/nerfactory/cuda/csrc/volume_rendering.cu index 8dc0b4635..db0a2b30b 100644 --- a/nerfactory/cuda/csrc/volume_rendering.cu +++ b/nerfactory/cuda/csrc/volume_rendering.cu @@ -35,24 +35,24 @@ __global__ void volumetric_rendering_forward_kernel( // accumulated rendering scalar_t T = 1.f; - scalar_t EPSILON = 1e-4f; - int j = 0; + scalar_t EPSILON = 1e-4f; + int j = 0; for (; j < numsteps; ++j) { - if (T < EPSILON) { - break; - } + if (T < EPSILON) { + break; + } const scalar_t delta = ends[j] - starts[j]; const scalar_t t = (ends[j] + starts[j]) * 0.5f; - const scalar_t alpha = 1.f - __expf(-sigmas[j] * delta); - const scalar_t weight = alpha * T; - accumulated_weight[0] += weight; + const scalar_t alpha = 1.f - __expf(-sigmas[j] * delta); + const scalar_t weight = alpha * T; + accumulated_weight[0] += weight; accumulated_depth[0] += weight * t; accumulated_color[0] += weight * rgbs[j * 3 + 0]; accumulated_color[1] += weight * rgbs[j * 3 + 1]; accumulated_color[2] += weight * rgbs[j * 3 + 2]; - T *= (1.f - alpha); - } + T *= (1.f - alpha); + } mask[0] = true; } @@ -100,25 +100,25 @@ __global__ void volumetric_rendering_backward_kernel( // backward of accumulated rendering scalar_t T = 1.f; - scalar_t EPSILON = 1e-4f; - int j = 0; + scalar_t EPSILON = 1e-4f; + int j = 0; scalar_t r = 0, g = 0, b = 0, d = 0; for (; j < numsteps; ++j) { - if (T < EPSILON) { - break; - } + if (T < EPSILON) { + break; + } const scalar_t delta = ends[j] - starts[j]; const scalar_t t = (ends[j] + starts[j]) * 0.5f; - const scalar_t alpha = 1.f - __expf(-sigmas[j] * delta); - const scalar_t weight = alpha * T; + const scalar_t alpha = 1.f - __expf(-sigmas[j] * delta); + const scalar_t weight = alpha * T; r += weight * rgbs[j * 3 + 0]; g += weight * rgbs[j * 3 + 1]; b += weight * rgbs[j * 3 + 2]; d += weight * t; - T *= (1.f - alpha); + T *= (1.f - alpha); grad_rgbs[j * 3 + 0] = grad_color[0] * weight; grad_rgbs[j * 3 + 1] = grad_color[1] * weight; @@ -131,7 +131,7 @@ __global__ void volumetric_rendering_backward_kernel( grad_weight[0] * (1.f - accumulated_weight[0]) + grad_depth[0] * (t * T - (accumulated_depth[0] - d)) ); - } + } } /** diff --git a/nerfactory/fields/density_fields/density_grid.py b/nerfactory/fields/density_fields/density_grid.py index 4d5e5bf8c..54bae2006 100644 --- a/nerfactory/fields/density_fields/density_grid.py +++ b/nerfactory/fields/density_fields/density_grid.py @@ -22,6 +22,7 @@ from torch import nn from torchtyping import TensorType import nerfactory.cuda as nerfactory_cuda +from nerfactory.cameras.rays import RaySamples def _create_grid_coords(resolution: int, device: torch.device = "cpu") -> TensorType["n_coords", 3]: @@ -44,9 +45,25 @@ def _create_grid_coords(resolution: int, device: torch.device = "cpu") -> Tensor class DensityGrid(nn.Module): """Cascaded multi-res density grids. - The multi-res grids are all centerd at [center, center, center] in the world space. - The first grid covers regions (center - base_scale / 2, center + base_scale / 2) - and the followings grows by 2x scale each. + The DensityGrid contains multi-level (`num_cascades`) grids from finest (0) to the coarsest (num_cascades-1). + All levels of grid have the same `resolution` but cover different regions in the space. The finest grid is the + smallest one, which has scale of `base_scale` and centered at `center`. Every next level of grid grows by 2x + on the scale but all shares the same center. + + For example: + - `center=0`, `base_scale=3`, `num_cascades=1` would create one grid (lvl=0) covers [-1.5, -1.5, -1.5, 1.5, 1.5, 1.5]. + - `center=0`, `base_scale=3`, `num_cascades=2` would create one grid (lvl=0) covers [-1.5, -1.5, -1.5, 1.5, 1.5, 1.5], + and one more grid (lvl=1) covers [-3, -3, -3, 3, 3, 3]. + + Normally you can simply set the `center` to the scene center, and `base_scale` to be the scene scale, and use + `num_cascades=1` to create a single level density grid that covers the scene. + + If, for example, you cares more about the central region of the scene and want to have more fine-grained density + grid at central region, you can set the `base_scale` to be the scale of the central region that you care. And adjust + `num_cascades` so that the coarsest level grid covers your entire scene. You should follow this rule so that the + multi-level grids can cover your scene: + + scene_scale <= base_scale * 2 ** (num_cascades - 1) TODO(ruilongli): support `center` and `base_scale` with 3-dim. @@ -86,6 +103,9 @@ class DensityGrid(nn.Module): self.register_buffer("grid_coords", grid_coords) self.register_buffer("grid_indices", grid_indices) + mean_density = torch.zeros((1,)) + self.register_buffer("mean_density", mean_density) + @torch.no_grad() def get_all_cells(self) -> List[Tuple[TensorType["n_cells"], TensorType["n_cells", 3]]]: """Returns all cells of the grid. @@ -171,23 +191,39 @@ class DensityGrid(nn.Module): # ema update valid_mask = (self.density_grid >= 0) & (tmp_grid >= 0) self.density_grid[valid_mask] = torch.maximum(self.density_grid[valid_mask] * decay, tmp_grid[valid_mask]) - mean_density = self.density_grid.mean().item() + self.mean_density.data = self.density_grid.mean() # pack to bitfield - self.density_bitfield.data = nerfactory_cuda.packbits(self.density_grid, min(mean_density, density_threshold)) + self.density_bitfield.data = nerfactory_cuda.packbits( + self.density_grid, min(self.mean_density.item(), density_threshold) + ) # TODO(ruilongli): max pooling? https://github.com/NVlabs/instant-ngp/blob/master/src/testbed_nerf.cu#L578 @torch.no_grad() - def get_densities(self, xyzs: TensorType[..., 3]) -> TensorType[..., 1]: - """Trilinear interpolation to get the density values. + def get_densities(self, ray_samples: RaySamples) -> TensorType[..., 1]: + """Get the density values stored in the density grid. Args: - xyzs (TensorType[..., 3]): 3D querry coordinate + ray_samples RaySamples: ray samples Returns: TensorType[..., 1]: Density values """ - # TODO(ruilongli) pass in RaySamples because we need dt to decide mip level. - raise NotImplementedError("We haven't implement the logic of querying densities from cascaded grid.") + positions = ray_samples.frustums.get_positions() + deltas = ray_samples.frustums.ends - ray_samples.frustums.starts + + _mip_levels, indices, _occupancies = nerfactory_cuda.occupancy_query( + positions.reshape(-1, 3), + deltas.reshape(-1), + self.center, + self.base_scale, + self.num_cascades, + self.resolution, + self.density_bitfield, + ) + + densities = self.density_grid.reshape(-1)[indices.long()] + densities = densities.view(*positions.shape[:-1], 1) + return densities def forward(self, x): """Not implemented.""" diff --git a/nerfactory/graphs/base.py b/nerfactory/graphs/base.py index a811f5ea1..9824d02eb 100644 --- a/nerfactory/graphs/base.py +++ b/nerfactory/graphs/base.py @@ -80,9 +80,10 @@ class Graph(AbstractGraph): intrinsics (torch.Tensor): Camera intrinsics. camera_to_world (torch.Tensor): Camera to world transformation. loss_coefficients (DictConfig): Loss specific weights. - steps_per_density_grid_update (int): How often to update density grid. scene_bounds (SceneBounds): Bounds of target scene. collider_config (DictConfig): Configuration of scene collider. + enable_density_field (bool): Whether to create a density field to filter samples. + density_field_config (DictConfig): Configuration of density field. """ def __init__( @@ -90,9 +91,10 @@ class Graph(AbstractGraph): intrinsics: torch.Tensor = None, camera_to_world: torch.Tensor = None, loss_coefficients: DictConfig = None, - steps_per_density_grid_update: int = 16, scene_bounds: SceneBounds = None, collider_config: DictConfig = None, + enable_density_field: bool = False, + density_field_config: DictConfig = None, **kwargs, ) -> None: super().__init__() @@ -102,10 +104,13 @@ class Graph(AbstractGraph): self.scene_bounds = scene_bounds self.collider_config = collider_config self.loss_coefficients = loss_coefficients - self.steps_per_density_grid_update = steps_per_density_grid_update + self.enable_density_field = enable_density_field + self.density_field_config = density_field_config + self.density_field = None self.kwargs = kwargs self.collider = None self.ray_generator = RayGenerator(self.intrinsics, self.camera_to_world) + self.populate_density_field() self.populate_collider() self.populate_fields() self.populate_misc_modules() # populate the modules @@ -118,6 +123,11 @@ class Graph(AbstractGraph): """Option to register callback for training functions""" self.callbacks = [] + def populate_density_field(self): + """Set the scene density field to use.""" + if self.enable_density_field: + self.density_field = instantiate_from_dict_config(self.density_field_config) + def populate_collider(self): """Set the scene bounds collider to use.""" self.collider = instantiate_from_dict_config(self.collider_config, scene_bounds=self.scene_bounds) diff --git a/nerfactory/graphs/modules/ray_sampler.py b/nerfactory/graphs/modules/ray_sampler.py index db9771df4..34db89903 100644 --- a/nerfactory/graphs/modules/ray_sampler.py +++ b/nerfactory/graphs/modules/ray_sampler.py @@ -32,7 +32,7 @@ class Sampler(nn.Module): """Generate Samples""" def __init__( - self, num_samples: int, density_field: Optional[DensityGrid] = None, weight_threshold: float = 1e-4 + self, num_samples: int, density_field: Optional[DensityGrid] = None, weight_threshold: float = 0.01 ) -> None: super().__init__() self.num_samples = num_samples @@ -47,11 +47,11 @@ class Sampler(nn.Module): """Generate ray samples with optional density filtering""" ray_samples = self.generate_ray_samples(*args, **kwargs) if self.density_field is not None: - densities = self.density_field.get_densities(ray_samples.frustums.get_positions()) - weights = ray_samples.get_weights(densities) - - valid_mask = weights >= self.weight_threshold - ray_samples.set_valid_mask(valid_mask) + densities = self.density_field.get_densities(ray_samples) + deltas = torch.clamp(ray_samples.frustums.ends - ray_samples.frustums.starts, min=1e-10) + density_threshold = torch.clamp(self.weight_threshold / deltas, max=self.density_field.mean_density) + valid_mask = densities > density_threshold + ray_samples.set_valid_mask(valid_mask & ray_samples.valid_mask) return ray_samples @@ -65,7 +65,7 @@ class SpacedSampler(Sampler): spacing_fn_inv: Callable, train_stratified=True, density_field: Optional[DensityGrid] = None, - weight_threshold: float = 1e-4, + weight_threshold: float = 1e-2, ) -> None: """ Args: @@ -129,7 +129,7 @@ class UniformSampler(SpacedSampler): num_samples: int, train_stratified=True, density_field: Optional[DensityGrid] = None, - weight_threshold: float = 1e-4, + weight_threshold: float = 1e-2, ) -> None: """ Args: @@ -157,7 +157,7 @@ class LinearDisparitySampler(SpacedSampler): num_samples: int, train_stratified=True, density_field: Optional[DensityGrid] = None, - weight_threshold: float = 1e-4, + weight_threshold: float = 1e-2, ) -> None: """ Args: @@ -185,7 +185,7 @@ class SqrtSampler(SpacedSampler): num_samples: int, train_stratified=True, density_field: Optional[DensityGrid] = None, - weight_threshold: float = 1e-4, + weight_threshold: float = 1e-2, ) -> None: """ Args: @@ -213,7 +213,7 @@ class LogSampler(SpacedSampler): num_samples: int, train_stratified=True, density_field: Optional[DensityGrid] = None, - weight_threshold: float = 1e-4, + weight_threshold: float = 1e-2, ) -> None: """ Args: @@ -242,7 +242,7 @@ class PDFSampler(Sampler): train_stratified: bool = True, include_original: bool = True, density_field: DensityGrid = None, - weight_threshold: float = 1e-4, + weight_threshold: float = 1e-2, histogram_padding: float = 0.01, ) -> None: """ diff --git a/nerfactory/graphs/vanilla_nerf.py b/nerfactory/graphs/vanilla_nerf.py index e7f664655..fdf7ba6fc 100644 --- a/nerfactory/graphs/vanilla_nerf.py +++ b/nerfactory/graphs/vanilla_nerf.py @@ -20,6 +20,7 @@ Implementation of vanilla nerf. from typing import Dict, List import torch +from omegaconf import DictConfig from torch.nn import Parameter from torchmetrics import PeakSignalNoiseRatio from torchmetrics.functional import structural_similarity_index_measure @@ -38,6 +39,7 @@ from nerfactory.renderers.renderers import ( RGBRenderer, ) from nerfactory.utils import colors, visualization, writer +from nerfactory.utils.callbacks import Callback class NeRFGraph(Graph): @@ -50,6 +52,8 @@ class NeRFGraph(Graph): far_plane (float, optional): Where to stop sampling points. Defaults to a distance of 6, num_coarse_samples (int, optional): Number of samples in coarse field evaluation. Defaults to 64, num_importance_samples(int, optional): Number of samples in fine field evaluation. Defaults to 64, + enable_density_field (bool): Whether to create a density field to filter samples. + density_field_config (DictConfig): Configuration of density field. """ def __init__( @@ -60,6 +64,8 @@ class NeRFGraph(Graph): far_plane: float = 6.0, num_coarse_samples: int = 64, num_importance_samples: int = 128, + enable_density_field: bool = False, + density_field_config: DictConfig = None, **kwargs, ) -> None: self.near_plane = near_plane @@ -68,7 +74,25 @@ class NeRFGraph(Graph): self.num_importance_samples = num_importance_samples self.field_coarse = None self.field_fine = None - super().__init__(intrinsics=intrinsics, camera_to_world=camera_to_world, **kwargs) + super().__init__( + intrinsics=intrinsics, + camera_to_world=camera_to_world, + enable_density_field=enable_density_field, + density_field_config=density_field_config, + **kwargs, + ) + + def register_callbacks(self) -> None: + """defining callbacks to run after every training iteration""" + self.callbacks = [] + if self.density_field is not None: + self.callbacks += [ + Callback( + self.density_field.update_every_num_iters, + self.density_field.update_density_grid, + density_eval_func=self.field_coarse.density_fn, + ) + ] def populate_fields(self): """Set the fields.""" @@ -85,8 +109,8 @@ class NeRFGraph(Graph): def populate_misc_modules(self): # samplers - self.sampler_uniform = UniformSampler(num_samples=self.num_coarse_samples) - self.sampler_pdf = PDFSampler(num_samples=self.num_importance_samples) + self.sampler_uniform = UniformSampler(num_samples=self.num_coarse_samples, density_field=self.density_field) + self.sampler_pdf = PDFSampler(num_samples=self.num_importance_samples, density_field=self.density_field) # renderers self.renderer_rgb = RGBRenderer(background_color=colors.WHITE) diff --git a/nerfactory/utils/config.py b/nerfactory/utils/config.py index 6b99a9a0e..6f1044054 100644 --- a/nerfactory/utils/config.py +++ b/nerfactory/utils/config.py @@ -103,6 +103,8 @@ class GraphConfig: loss_coefficients: Dict[str, Any] = MISSING # additional optional parameters here field_implementation: Optional[str] = "torch" + enable_density_field: Optional[bool] = False + density_field_config: Dict[str, Any] = MISSING @dataclass