diff --git a/configs/graph_instant_ngp.yaml b/configs/graph_instant_ngp.yaml index 2cada5734..b01a76c60 100644 --- a/configs/graph_instant_ngp.yaml +++ b/configs/graph_instant_ngp.yaml @@ -21,7 +21,8 @@ graph: field_implementation: tcnn # torch, tcnn, ... loss_coefficients: # loss coefficients rgb_loss: 1.0 - + enable_collider: false + # optimizer options for network optimizers: fields: diff --git a/nerfactory/cuda/__init__.py b/nerfactory/cuda/__init__.py index e6347705d..85799f4ab 100644 --- a/nerfactory/cuda/__init__.py +++ b/nerfactory/cuda/__init__.py @@ -33,9 +33,9 @@ class VolumeRenderer(torch.autograd.Function): @staticmethod @custom_fwd(cast_inputs=torch.float32) - def forward(ctx, packed_info, starts, ends, sigmas, rgbs): + def forward(ctx, packed_info, starts, ends, sigmas, rgbs, opacities): accumulated_weight, accumulated_depth, accumulated_color, mask = volumetric_rendering_forward( - packed_info, starts, ends, sigmas, rgbs + packed_info, starts, ends, sigmas, rgbs, opacities ) # TODO(ruilongli): accelerate for torch.no_grad()? ctx.save_for_backward( @@ -47,6 +47,7 @@ class VolumeRenderer(torch.autograd.Function): ends, sigmas, rgbs, + opacities, ) return accumulated_weight, accumulated_depth, accumulated_color, mask @@ -62,6 +63,7 @@ class VolumeRenderer(torch.autograd.Function): ends, sigmas, rgbs, + opacities, ) = ctx.saved_tensors grad_sigmas, grad_rgbs = volumetric_rendering_backward( accumulated_weight, @@ -75,9 +77,10 @@ class VolumeRenderer(torch.autograd.Function): ends, sigmas, rgbs, + opacities, ) # corresponds to the input argument list of forward() - return None, None, None, grad_sigmas, grad_rgbs + return None, None, None, grad_sigmas, grad_rgbs, None def unpackbits(x: torch.Tensor) -> torch.Tensor: diff --git a/nerfactory/cuda/csrc/nerfactory.cpp b/nerfactory/cuda/csrc/nerfactory.cpp index 569c47306..38a39e91c 100644 --- a/nerfactory/cuda/csrc/nerfactory.cpp +++ b/nerfactory/cuda/csrc/nerfactory.cpp @@ -28,8 +28,9 @@ std::vector raymarching( const int grid_size, const torch::Tensor grid_bitfield, // sampling args + const int marching_steps, const int max_total_samples, - const int num_steps, + const int max_ray_samples, const float cone_angle, const float step_scale ) {}; @@ -39,7 +40,8 @@ std::vector volumetric_rendering_forward( torch::Tensor starts, torch::Tensor ends, torch::Tensor sigmas, - torch::Tensor rgbs + torch::Tensor rgbs, + torch::Tensor opacities ) {}; std::vector volumetric_rendering_backward( @@ -53,7 +55,8 @@ std::vector volumetric_rendering_backward( torch::Tensor starts, torch::Tensor ends, torch::Tensor sigmas, - torch::Tensor rgbs + torch::Tensor rgbs, + torch::Tensor opacities ) {}; std::vector occupancy_query( diff --git a/nerfactory/cuda/csrc/nerfactory.cu b/nerfactory/cuda/csrc/nerfactory.cu index 6142ceaae..1b328b068 100644 --- a/nerfactory/cuda/csrc/nerfactory.cu +++ b/nerfactory/cuda/csrc/nerfactory.cu @@ -28,8 +28,9 @@ std::vector raymarching( const int grid_size, const torch::Tensor grid_bitfield, // sampling args + const int marching_steps, const int max_total_samples, - const int num_steps, + const int max_ray_samples, const float cone_angle, const float step_scale ); @@ -39,7 +40,8 @@ std::vector volumetric_rendering_forward( torch::Tensor starts, torch::Tensor ends, torch::Tensor sigmas, - torch::Tensor rgbs + torch::Tensor rgbs, + torch::Tensor opacities ); std::vector volumetric_rendering_backward( @@ -53,7 +55,8 @@ std::vector volumetric_rendering_backward( torch::Tensor starts, torch::Tensor ends, torch::Tensor sigmas, - torch::Tensor rgbs + torch::Tensor rgbs, + torch::Tensor opacities ); std::vector occupancy_query( diff --git a/nerfactory/cuda/csrc/raymarching.cu b/nerfactory/cuda/csrc/raymarching.cu index d6c53629b..9cd5bfabe 100644 --- a/nerfactory/cuda/csrc/raymarching.cu +++ b/nerfactory/cuda/csrc/raymarching.cu @@ -1,8 +1,8 @@ #include "include/helpers_cuda.h" -inline __device__ float min_step_size(uint32_t num_steps) { - return __SQRT3() / num_steps; +inline __device__ float min_step_size(uint32_t max_ray_samples) { + return __SQRT3() / max_ray_samples; } // Maximum step size is the width of the coarsest gridsize cell. @@ -100,6 +100,10 @@ inline __device__ float advance_to_next_voxel( return t; } + +// Two modes: +// 1. "vertically" marching: a portion of rays march maximum steps. (during training) +// 2. "horizontally" marching: all rays march together for certain steps. (during inference) template __global__ void kernel_raymarching( // rays info @@ -115,11 +119,12 @@ __global__ void kernel_raymarching( const int grid_size, const uint8_t* grid_bitfield, // sampling + const int marching_steps, // used only for inference. -1 means disable it. const int max_total_samples, - const int num_steps, // default 1024 + const int max_ray_samples, // default 1024 const float cone_angle, // default 0. for nerf-syn and 1/256 for large scene const float step_scale, - int* numsteps_counter, + int* steps_counter, int* rays_counter, int* packed_info_out, scalar_t* origins_out, @@ -142,60 +147,65 @@ __global__ void kernel_raymarching( // TODO(ruilongli): perturb `near` as in ngp_pl? // TODO(ruilongli): pre-compute `dt_min`, `dt_max` as it is a constant? - float dt_min = min_step_size(num_steps) * step_scale; + float dt_min = min_step_size(max_ray_samples) * step_scale; float dt_max = max_step_size(grid_cascades, grid_size) * grid_scale; - - // first pass to compute an accurate number of steps - uint32_t j = 0; - float t0 = near; - float dt = calc_dt(t0, cone_angle, dt_min, dt_max); - float t1 = t0 + dt; - float t_mid = (t0 + t1) * 0.5f; + + uint32_t ray_idx, base, marching_samples; + uint32_t j; + float t0, dt, t1, t_mid; + if (marching_steps < 0) { + // first pass to compute an accurate number of steps + j = 0; + t0 = near; + dt = calc_dt(t0, cone_angle, dt_min, dt_max); + t1 = t0 + dt; + t_mid = (t0 + t1) * 0.5f; - while (t_mid < far && j < num_steps) { - // current center - const float x = ox + t_mid * dx; - const float y = oy + t_mid * dy; - const float z = oz + t_mid * dz; - uint32_t mip = mip_from_dt(x, y, z, grid_cascades, dt, grid_size, grid_center, grid_scale); - - if (grid_occupied_at(x, y, z, grid_bitfield, mip, grid_size, grid_center, grid_scale)) { - ++j; - // march to next sample - t0 = t1; - dt = calc_dt(t0, cone_angle, dt_min, dt_max); - t1 = t0 + dt; - t_mid = (t0 + t1) * 0.5f; - } - else { - // march to next sample - float res = (grid_size >> mip) * grid_scale; - t_mid = advance_to_next_voxel( - t_mid, x, y, z, dx, dy, dz, rdx, rdy, rdz, res, dt_min - ); - dt = calc_dt(t_mid, cone_angle, dt_min, dt_max); - t0 = t_mid - dt * 0.5f; - t1 = t_mid + dt * 0.5f; + while (t_mid < far && j < max_ray_samples) { + // current center + const float x = ox + t_mid * dx; + const float y = oy + t_mid * dy; + const float z = oz + t_mid * dz; + uint32_t mip = mip_from_dt(x, y, z, grid_cascades, dt, grid_size, grid_center, grid_scale); + + if (grid_occupied_at(x, y, z, grid_bitfield, mip, grid_size, grid_center, grid_scale)) { + ++j; + // march to next sample + t0 = t1; + dt = calc_dt(t0, cone_angle, dt_min, dt_max); + t1 = t0 + dt; + t_mid = (t0 + t1) * 0.5f; + } + else { + // march to next sample + float res = (grid_size >> mip) * grid_scale; + t_mid = advance_to_next_voxel( + t_mid, x, y, z, dx, dy, dz, rdx, rdy, rdz, res, dt_min + ); + dt = calc_dt(t_mid, cone_angle, dt_min, dt_max); + t0 = t_mid - dt * 0.5f; + t1 = t_mid + dt * 0.5f; + } } + if (j == 0) return; + + marching_samples = j; + base = atomicAdd(steps_counter, marching_samples); + if (base + marching_samples > max_total_samples) return; + ray_idx = atomicAdd(rays_counter, 1); + + } else { + marching_samples = marching_steps; + base = i * marching_steps; + ray_idx = i; } - if (j == 0) return; - uint32_t numsteps = j; - uint32_t base = atomicAdd(numsteps_counter, numsteps); - if (base + numsteps > max_total_samples) return; - // locate origins_out += base * 3; dirs_out += base * 3; starts_out += base; ends_out += base; - uint32_t ray_idx = atomicAdd(rays_counter, 1); - - packed_info_out[ray_idx * 3 + 0] = i; // ray idx in {rays_o, rays_d} - packed_info_out[ray_idx * 3 + 1] = base; // point idx start. - packed_info_out[ray_idx * 3 + 2] = numsteps; // point idx shift. - // Second round j = 0; t0 = near; @@ -203,7 +213,7 @@ __global__ void kernel_raymarching( t1 = t0 + dt; t_mid = (t0 + t1) / 2.; - while (t_mid < far && j < num_steps) { + while (t_mid < far && j < marching_samples) { // current center const float x = ox + t_mid * dx; const float y = oy + t_mid * dy; @@ -235,8 +245,13 @@ __global__ void kernel_raymarching( dt = calc_dt(t_mid, cone_angle, dt_min, dt_max); t0 = t_mid - dt * 0.5f; t1 = t_mid + dt * 0.5f; - } - } + } + } + + packed_info_out[ray_idx * 3 + 0] = i; // ray idx in {rays_o, rays_d} + packed_info_out[ray_idx * 3 + 1] = base; // point idx start. + packed_info_out[ray_idx * 3 + 2] = j; // point idx shift (actual marching samples). + return; } @@ -302,8 +317,9 @@ __global__ void kernel_occupancy_query( * @param grid_cascades Density grid levels. * @param grid_size Density grid resolution. * @param grid_bitfield Density grid uint8 bit field. + * @param marching_steps Marching steps during inference. * @param max_total_samples Maximum total number of samples in this batch. - * @param num_steps Used to define the minimal step size: SQRT3() / num_steps. + * @param max_ray_samples Used to define the minimal step size: SQRT3() / max_ray_samples. * @param cone_angle 0. for nerf-synthetic and 1./256 for real scenes. * @param step_scale Scale up the step size by this much. Usually equals to scene scale. * @return std::vector @@ -330,8 +346,9 @@ std::vector raymarching( const int grid_size, const torch::Tensor grid_bitfield, // sampling args + const int marching_steps, const int max_total_samples, - const int num_steps, + const int max_ray_samples, const float cone_angle, const float step_scale ) { @@ -349,18 +366,20 @@ std::vector raymarching( const int blocks = CUDA_N_BLOCKS_NEEDED(n_rays, threads); // helper counter - torch::Tensor numsteps_counter = torch::zeros( + torch::Tensor steps_counter = torch::zeros( {1}, rays_o.options().dtype(torch::kInt32)); torch::Tensor rays_counter = torch::zeros( {1}, rays_o.options().dtype(torch::kInt32)); + int output_total_samples = marching_steps < 0 ? max_total_samples : marching_steps * n_rays; + // output samples torch::Tensor packed_info = torch::zeros( {n_rays, 3}, rays_o.options().dtype(torch::kInt32)); // ray_id, sample_id, num_samples - torch::Tensor origins = torch::zeros({max_total_samples, 3}, rays_o.options()); - torch::Tensor dirs = torch::zeros({max_total_samples, 3}, rays_o.options()); - torch::Tensor starts = torch::zeros({max_total_samples, 1}, rays_o.options()); - torch::Tensor ends = torch::zeros({max_total_samples, 1}, rays_o.options()); + torch::Tensor origins = torch::zeros({output_total_samples, 3}, rays_o.options()); + torch::Tensor dirs = torch::zeros({output_total_samples, 3}, rays_o.options()); + torch::Tensor starts = torch::zeros({output_total_samples, 1}, rays_o.options()); + torch::Tensor ends = torch::zeros({output_total_samples, 1}, rays_o.options()); AT_DISPATCH_FLOATING_TYPES_AND_HALF( rays_o.scalar_type(), @@ -380,11 +399,12 @@ std::vector raymarching( grid_size, grid_bitfield.data_ptr(), // sampling args + marching_steps, // used only for inference. -1 means disable it. max_total_samples, - num_steps, + max_ray_samples, cone_angle, step_scale, - numsteps_counter.data_ptr(), // total samples. + steps_counter.data_ptr(), // total samples. rays_counter.data_ptr(), // total rays. packed_info.data_ptr(), origins.data_ptr(), diff --git a/nerfactory/cuda/csrc/volume_rendering.cu b/nerfactory/cuda/csrc/volume_rendering.cu index db0a2b30b..996dacbdd 100644 --- a/nerfactory/cuda/csrc/volume_rendering.cu +++ b/nerfactory/cuda/csrc/volume_rendering.cu @@ -9,6 +9,7 @@ __global__ void volumetric_rendering_forward_kernel( const scalar_t* ends, // input end t const scalar_t* sigmas, // input density after activation const scalar_t* rgbs, // input rgb after activation + const scalar_t* opacities, // input opacities for each ray // should be all-zero initialized scalar_t* accumulated_weight, // output scalar_t* accumulated_depth, // output @@ -32,9 +33,10 @@ __global__ void volumetric_rendering_forward_kernel( accumulated_depth += i; accumulated_color += i * 3; mask += i; - + opacities += i; + // accumulated rendering - scalar_t T = 1.f; + scalar_t T = 1.f - opacities[0]; scalar_t EPSILON = 1e-4f; int j = 0; for (; j < numsteps; ++j) { @@ -65,6 +67,7 @@ __global__ void volumetric_rendering_backward_kernel( const scalar_t* ends, // input end t const scalar_t* sigmas, // input density after activation const scalar_t* rgbs, // input rgb after activation + const scalar_t* opacities, // input opacities for each ray const scalar_t* accumulated_weight, // forward output const scalar_t* accumulated_depth, // forward output const scalar_t* accumulated_color, // forward output @@ -93,13 +96,14 @@ __global__ void volumetric_rendering_backward_kernel( accumulated_weight += i; accumulated_depth += i; accumulated_color += i * 3; + opacities += i; grad_weight += i; grad_depth += i; grad_color += i * 3; // backward of accumulated rendering - scalar_t T = 1.f; + scalar_t T = 1.f - opacities[0]; scalar_t EPSILON = 1e-4f; int j = 0; scalar_t r = 0, g = 0, b = 0, d = 0; @@ -148,6 +152,7 @@ __global__ void volumetric_rendering_backward_kernel( * @param ends: Where the frustum-shape sample ends along a ray. [total_samples, 1] * @param sigmas Densities at those samples. [total_samples, 1] * @param rgbs RGBs at those samples. [total_samples, 3] + * @param opacities Opacities that the rays already have. Zero for training but non-zero for eval. [total_samples, 1] * @return std::vector * - accumulated_weight: Ray opacity. [n_rays, 1] * - accumulated_depth: Ray depth. [n_rays, 1] @@ -159,7 +164,8 @@ std::vector volumetric_rendering_forward( torch::Tensor starts, torch::Tensor ends, torch::Tensor sigmas, - torch::Tensor rgbs + torch::Tensor rgbs, + torch::Tensor opacities ) { DEVICE_GUARD(packed_info); CHECK_INPUT(packed_info); @@ -167,11 +173,13 @@ std::vector volumetric_rendering_forward( CHECK_INPUT(ends); CHECK_INPUT(sigmas); CHECK_INPUT(rgbs); + CHECK_INPUT(opacities); TORCH_CHECK(packed_info.ndimension() == 2 & packed_info.size(1) == 3); TORCH_CHECK(starts.ndimension() == 2 & starts.size(1) == 1); TORCH_CHECK(ends.ndimension() == 2 & ends.size(1) == 1); TORCH_CHECK(sigmas.ndimension() == 2 & sigmas.size(1) == 1); TORCH_CHECK(rgbs.ndimension() == 2 & rgbs.size(1) == 3); + TORCH_CHECK(opacities.ndimension() == 2 & opacities.size(1) == 1); const uint32_t n_rays = packed_info.size(0); @@ -196,6 +204,7 @@ std::vector volumetric_rendering_forward( ends.data_ptr(), sigmas.data_ptr(), rgbs.data_ptr(), + opacities.data_ptr(), accumulated_weight.data_ptr(), accumulated_depth.data_ptr(), accumulated_color.data_ptr(), @@ -222,7 +231,8 @@ std::vector volumetric_rendering_backward( torch::Tensor starts, torch::Tensor ends, torch::Tensor sigmas, - torch::Tensor rgbs + torch::Tensor rgbs, + torch::Tensor opacities ) { DEVICE_GUARD(packed_info); const uint32_t n_rays = packed_info.size(0); @@ -245,6 +255,7 @@ std::vector volumetric_rendering_backward( ends.data_ptr(), sigmas.data_ptr(), rgbs.data_ptr(), + opacities.data_ptr(), accumulated_weight.data_ptr(), accumulated_depth.data_ptr(), accumulated_color.data_ptr(), diff --git a/nerfactory/fields/density_fields/density_grid.py b/nerfactory/fields/density_fields/density_grid.py index 54bae2006..b235547aa 100644 --- a/nerfactory/fields/density_fields/density_grid.py +++ b/nerfactory/fields/density_fields/density_grid.py @@ -103,7 +103,7 @@ class DensityGrid(nn.Module): self.register_buffer("grid_coords", grid_coords) self.register_buffer("grid_indices", grid_indices) - mean_density = torch.zeros((1,)) + mean_density = torch.zeros(()) self.register_buffer("mean_density", mean_density) @torch.no_grad() diff --git a/nerfactory/graphs/base.py b/nerfactory/graphs/base.py index 130f65966..a13e89645 100644 --- a/nerfactory/graphs/base.py +++ b/nerfactory/graphs/base.py @@ -171,24 +171,32 @@ class Graph(AbstractGraph): """Run forward starting with a ray bundle.""" if self.collider is not None: intersected_ray_bundle = self.collider(ray_bundle) + valid_mask = intersected_ray_bundle.valid_mask[..., 0] else: # NOTE(ruilongli): we don't need collider for ngp intersected_ray_bundle = ray_bundle + valid_mask = None if batch is None: # during inference, keep all rays outputs = self.get_outputs(intersected_ray_bundle) return outputs - # during training, keep only the rays that intersect the scene. discard the rest - valid_mask = intersected_ray_bundle.valid_mask[..., 0] - masked_intersected_ray_bundle = intersected_ray_bundle[valid_mask] - masked_batch = get_masked_dict(batch, valid_mask) # NOTE(ethan): this is really slow if on CPU! - outputs = self.get_outputs(masked_intersected_ray_bundle) - metrics_dict = self.get_metrics_dict(outputs=outputs, batch=masked_batch) + if valid_mask is not None: + intersected_ray_bundle = intersected_ray_bundle[valid_mask] + # during training, keep only the rays that intersect the scene. discard the rest + batch = get_masked_dict(batch, valid_mask) # NOTE(ethan): this is really slow if on CPU! + + outputs = self.get_outputs(intersected_ray_bundle) + metrics_dict = self.get_metrics_dict(outputs=outputs, batch=batch) loss_dict = self.get_loss_dict( - outputs=outputs, batch=masked_batch, metrics_dict=metrics_dict, loss_coefficients=self.loss_coefficients + outputs=outputs, batch=batch, metrics_dict=metrics_dict, loss_coefficients=self.loss_coefficients ) + + # scaling losses by coefficients. + for loss_name in loss_dict.keys(): + if loss_name in self.loss_coefficients: + loss_dict[loss_name] *= self.loss_coefficients[loss_name] return outputs, loss_dict, metrics_dict def forward(self, ray_indices: TensorType["num_rays", 3], batch: Union[str, Dict[str, torch.tensor]] = None): diff --git a/nerfactory/graphs/instant_ngp.py b/nerfactory/graphs/instant_ngp.py index 6057fdd16..8f3f2056b 100644 --- a/nerfactory/graphs/instant_ngp.py +++ b/nerfactory/graphs/instant_ngp.py @@ -86,31 +86,99 @@ class NGPGraph(Graph): # TODO(ruilongli) # - train test difference # - visualize "depth_density_grid" - ray_samples, packed_info, t_min, t_max = self.sampler(ray_bundle, self.field.aabb) + num_rays = len(ray_bundle) + device = ray_bundle.origins.device - field_outputs = self.field.forward(ray_samples) - rgbs = field_outputs[FieldHeadNames.RGB] - sigmas = field_outputs[FieldHeadNames.DENSITY] + if self.training: + ray_samples, packed_info, t_min, t_max = self.sampler(ray_bundle, self.field.aabb) - accumulated_weight, accumulated_depth, accumulated_color, mask = nerfactory_cuda.VolumeRenderer.apply( - packed_info, ray_samples.frustums.starts, ray_samples.frustums.ends, sigmas, rgbs - ) - accumulated_depth = torch.clip(accumulated_depth, t_min[:, None], t_max[:, None]) - accumulated_color = accumulated_color + colors.WHITE.to(accumulated_color) * (1.0 - accumulated_weight) + field_outputs = self.field.forward(ray_samples) + rgbs = field_outputs[FieldHeadNames.RGB] + sigmas = field_outputs[FieldHeadNames.DENSITY] + + opacities = torch.zeros((num_rays, 1), device=device) + ( + accumulated_weight, + accumulated_depth, + accumulated_color, + alive_ray_mask, + ) = nerfactory_cuda.VolumeRenderer.apply( + packed_info, ray_samples.frustums.starts, ray_samples.frustums.ends, sigmas, rgbs, opacities + ) + accumulated_depth = torch.clip(accumulated_depth, t_min[:, None], t_max[:, None]) + accumulated_color = accumulated_color + colors.WHITE.to(accumulated_color) * (1.0 - accumulated_weight) + + else: + accumulated_weight = torch.zeros((num_rays, 1), device=device) + accumulated_depth = torch.zeros((num_rays, 1), device=device) + accumulated_color = torch.zeros((num_rays, 3), device=device) + alive_ray_mask = torch.ones((num_rays), device=device, dtype=torch.bool) # the ray we kept from sampler + + t_min, t_max = None, None + t_marching = None + samples = 0 + while samples < self.sampler.num_samples: + num_alive = alive_ray_mask.sum().item() + if num_alive == 0: + break + + min_samples = 1 + num_samples = max(min(num_rays // num_alive, 64), min_samples) + samples += num_samples + + ray_samples, packed_info, t_marching, t_max = self.sampler( + ray_bundle, self.field.aabb, marching_steps=num_samples, t_min=t_marching, t_max=t_max + ) + if t_min is None: + t_min = t_marching + + field_outputs = self.field.forward(ray_samples) + rgbs = field_outputs[FieldHeadNames.RGB] + sigmas = field_outputs[FieldHeadNames.DENSITY] + + ( + _accumulated_weight, + _accumulated_depth, + _accumulated_color, + _, + ) = nerfactory_cuda.VolumeRenderer.apply( + packed_info, + ray_samples.frustums.starts, + ray_samples.frustums.ends, + sigmas, + rgbs, + accumulated_weight, + ) + accumulated_weight += _accumulated_weight + accumulated_depth += _accumulated_depth + accumulated_color += _accumulated_color + + # all samples for this ray have been drawn + alive_ray_mask[packed_info[:, 2] < num_samples] = False + + # this ray has converged + alive_ray_mask[accumulated_weight[:, 0] > 1.0 - 1e-4] = False + + # march forward and skip those are not alive + t_marching = ray_samples.frustums.ends.reshape(num_rays, num_samples).max(dim=-1).values + t_marching[~alive_ray_mask] = 1e10 + + accumulated_depth = torch.clip(accumulated_depth, t_min[:, None], t_max[:, None]) + accumulated_color = accumulated_color + colors.WHITE.to(accumulated_color) * (1.0 - accumulated_weight) outputs = { "rgb": accumulated_color, "accumulation": accumulated_weight, "depth": accumulated_depth, - "mask": mask, # the rays we skipped from sampler + "alive_ray_mask": alive_ray_mask, # the rays we kept from sampler } return outputs def get_loss_dict(self, outputs, batch, metrics_dict, loss_coefficients): device = self.get_device() image = batch["image"].to(device) - if "mask" in outputs: - mask = outputs["mask"] + if "alive_ray_mask" in outputs: + mask = outputs["alive_ray_mask"] rgb_loss = self.rgb_loss(image[mask], outputs["rgb"][mask]) else: rgb_loss = self.rgb_loss(image, outputs["rgb"]) diff --git a/nerfactory/graphs/modules/ray_sampler.py b/nerfactory/graphs/modules/ray_sampler.py index 34db89903..1e464359b 100644 --- a/nerfactory/graphs/modules/ray_sampler.py +++ b/nerfactory/graphs/modules/ray_sampler.py @@ -357,6 +357,9 @@ class NGPSpacedSampler(Sampler): ray_bundle: RayBundle, aabb: TensorType[2, 3], num_samples: Optional[int] = None, + marching_steps: Optional[int] = 128, + t_min: Optional[TensorType["num_rays"]] = None, + t_max: Optional[TensorType["num_rays"]] = None, ) -> Tuple[RaySamples, TensorType["total_samples", 3], TensorType["total_samples"], TensorType["total_samples"]]: """Generate ray samples in a bounding box. @@ -379,13 +382,24 @@ class NGPSpacedSampler(Sampler): aabb = aabb.flatten() rays_o = ray_bundle.origins.contiguous() rays_d = ray_bundle.directions.contiguous() - t_min, t_max = nerfactory_cuda.ray_aabb_intersect(rays_o, rays_d, aabb) scene_scale = (aabb[3:6] - aabb[0:3]).max() - # TODO(ruilongli): * 16 is for original impl for training. Need to run - # some profiling test with this choice. - max_samples_per_batch = len(rays_o) * num_samples + if t_min is None or t_max is None: + # TODO(ruilongli): this clipping here is stupid. Try to avoid that. + t_min, t_max = nerfactory_cuda.ray_aabb_intersect(rays_o, rays_d, aabb) + t_min = torch.clamp(t_min, max=1e10) + t_max = torch.clamp(t_max, max=1e10) + # sampling arguments + if self.training: + # TODO(ruilongli): * 16 is for original impl for training. Need to run + # some profiling test with this choice. + marching_steps = -1 + max_samples_per_batch = len(rays_o) * num_samples + else: + max_samples_per_batch = len(rays_o) * marching_steps + + # marching packed_info, origins, dirs, starts, ends = nerfactory_cuda.raymarching( # rays rays_o, @@ -399,20 +413,28 @@ class NGPSpacedSampler(Sampler): self.density_field.resolution, self.density_field.density_bitfield, # sampling args + marching_steps, max_samples_per_batch, num_samples, 0.0, scene_scale, ) - total_samples = max(packed_info[:, -1].sum(), 1) - origins = origins[:total_samples] - dirs = dirs[:total_samples] - starts = starts[:total_samples] - ends = ends[:total_samples] + # squeeze valid samples + if self.training: + total_samples = max(packed_info[:, -1].sum(), 1) + origins = origins[:total_samples] + dirs = dirs[:total_samples] + starts = starts[:total_samples] + ends = ends[:total_samples] + valid_mask = None + else: + valid_mask = (dirs != 0.0).any(dim=-1, keepdim=True) + + # return samples zeros = torch.zeros_like(origins[:, :1]) - ray_samples = RaySamples( frustums=Frustums(origins=origins, directions=dirs, starts=starts, ends=ends, pixel_area=zeros), + valid_mask=valid_mask, ) return ray_samples, packed_info, t_min, t_max