Pipeline v2 (#258)

* Model class done, pipeline class still needs more visualization stuffs

* Renaming the BaseDataContainer to DataloaderOutputs and adding documentation

* Pushing more documentation, removing some trivial functions

* Updating dict typing

* Removing config dicts for collider

* Passing tests

* Pushing shell new dataloader

* Saving work, need to fix NGP model's get_loss_dict and Pipeline

* V1 of the ngp pipeline should be done. Need train script and debugging

* Updating with latest instant-ngp code

* Syntax updates from latest ngp graph code

* Linter, removing unneeded code in new NGP graph code

* Style

* Should be good now. Linter is happy and the code is verbatum current instant-ngp

* Everything done except novel view testing code

* Fixing some nitpicks

* Debugging dataloader

* merged master

* WOP. integrating pipeline with configs and notebook demo

* removing model folder

* a lot of name changing from Graph to Model

* in progress... changing trainer to use pipeline... broken right now

* pipeline code runs

* switched to val dataset

* makes run_eval.py work

* updated configs for pipeline change

* fixing test_run_train script

* fixing black and linting

* fix docs

* small updates to trainer

Co-authored-by: Jake Austin <jake-austin@berkeley.edu>
This commit is contained in:
Ethan WeberandJake Austin authored and GitHub committed 2022-08-10 14:59:45 -07:00
1 parent 670c2a01db
commit 0b042be956
55 files changed
+8595 -8263

No files matched your search

+13
View File
@@ -0,0 +1,13 @@
# Copyright 2022 The Plenoptix Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
@@ -0,0 +1,69 @@
# Copyright 2022 The Plenoptix Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Ray generator.
"""
from torch import nn
from torchtyping import TensorType
from nerfactory.cameras.cameras import get_camera_model
from nerfactory.cameras.rays import RayBundle
class RayGenerator(nn.Module):
"""torch.nn Module for generating rays.
This class will store the intrinsics and extrinsics parameters of the cameras.
Args:
intrinsics: The intrinsics parameters.
camera_to_world: Camera to world transformation matrix.
"""
def __init__(
self,
intrinsics: TensorType["num_cameras", "num_intrinsics_params"],
camera_to_world: TensorType["num_cameras", 3, 4],
) -> None:
super().__init__()
self.num_cameras, self.num_intrinsics_params = intrinsics.shape
assert self.num_cameras >= 0
self.intrinsics = nn.Parameter(intrinsics, requires_grad=False)
self.camera_to_world = nn.Parameter(camera_to_world, requires_grad=False)
# TODO(ethan): add learnable parameters that are deltas on the intrinsics and camera_to_world parameters
# NOTE(ethan): we currently assume all images have the same height and width
camera_index = 0
self.camera_class = get_camera_model(self.num_intrinsics_params)
camera = self.camera_class(*self.intrinsics[camera_index].tolist())
self.image_coords = nn.Parameter(camera.get_image_coords(), requires_grad=False)
def forward(self, ray_indices: TensorType["num_rays", 3]) -> RayBundle:
"""Index into the cameras to generate the rays.
Args:
ray_indices: Contains camera, row, and col indicies for target rays.
"""
c = ray_indices[:, 0] # camera indices
y = ray_indices[:, 1] # row indices
x = ray_indices[:, 2] # col indices
intrinsics = self.intrinsics[c]
camera_to_world = self.camera_to_world[c]
coords = self.image_coords[y, x]
ray_bundle = self.camera_class.generate_rays(
intrinsics=intrinsics, camera_to_world=camera_to_world, coords=coords
)
ray_bundle.camera_indices = c[..., None] # ["num_rays",1]
return ray_bundle
+55
View File
@@ -0,0 +1,55 @@
# Copyright 2022 The Plenoptix Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Collection of Ray Losses"""
import torch
from torchtyping import TensorType
from nerfactory.cameras.rays import RaySamples
def distortion_loss(
ray_samples: RaySamples, densities: TensorType["bs":..., "num_samples", 1]
) -> TensorType["bs":..., 1]:
"""Ray baserd distortion loss proposed in MipNeRF-360.
.. math::
\\mathcal{L}(\\mathbf{s}, \\mathbf{w}) =\\iint\\limits_{-\\infty}^{\\,\\,\\,\\infty}
\\mathbf{w}_\\mathbf{s}(u)\\mathbf{w}_\\mathbf{s}(v)|u - v|\\,d_{u}\\,d_{v}
where :math:`\\mathbf{w}_\\mathbf{s}(u)=\\sum_i w_i \\mathbb{1}_{[\\mathbf{s}_i, \\mathbf{s}_{i+1})}(u)`
is the weight at location :math:`u` between bin locations :math:`s_i` and :math:`s_{i+1}`.
Args:
ray_samples: Ray samples to compute loss over
densities: Predicted sample densities
Returns:
Distortion Loss.
"""
# Compute the weight at each sample location
weights = ray_samples.get_weights(densities)
midpoints = (ray_samples.frustums.starts + ray_samples.frustums.ends) / 2.0 # (..., num_samples, 1)
loss = (
weights * weights[..., None, :, 0] * torch.abs(midpoints - midpoints[..., None, :, 0])
) # (..., num_samples, num_samples)
loss = torch.sum(loss, dim=(-1, -2))[..., None] # (..., num_samples)
loss = loss + 1 / 3.0 * torch.sum(weights**2 * (ray_samples.frustums.ends - ray_samples.frustums.starts), dim=-2)
return loss
+432
View File
@@ -0,0 +1,432 @@
# Copyright 2022 The Plenoptix Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Collection of sampling strategies
"""
from abc import abstractmethod
from typing import Callable, Optional, Tuple
import torch
from torch import nn
from torchtyping import TensorType
import nerfactory.cuda as nerfactory_cuda
from nerfactory.cameras.rays import Frustums, RayBundle, RaySamples
from nerfactory.fields.density_fields.density_grid import DensityGrid
class Sampler(nn.Module):
"""Generate Samples"""
def __init__(
self, num_samples: int, density_field: Optional[DensityGrid] = None, weight_threshold: float = 0.01
) -> None:
super().__init__()
self.num_samples = num_samples
self.density_field = density_field
self.weight_threshold = weight_threshold
@abstractmethod
def generate_ray_samples(self) -> RaySamples:
"""Generate Ray Samples"""
def forward(self, *args, **kwargs) -> RaySamples:
"""Generate ray samples with optional density filtering"""
ray_samples = self.generate_ray_samples(*args, **kwargs)
if self.density_field is not None:
densities = self.density_field.get_densities(ray_samples)
deltas = torch.clamp(ray_samples.frustums.ends - ray_samples.frustums.starts, min=1e-10)
density_threshold = torch.clamp(self.weight_threshold / deltas, max=self.density_field.mean_density)
valid_mask = densities > density_threshold
ray_samples.set_valid_mask(valid_mask & ray_samples.valid_mask)
return ray_samples
class SpacedSampler(Sampler):
"""Sample points according to a function."""
def __init__(
self,
num_samples: int,
spacing_fn: Callable,
spacing_fn_inv: Callable,
train_stratified=True,
density_field: Optional[DensityGrid] = None,
weight_threshold: float = 1e-2,
) -> None:
"""
Args:
num_samples: Number of samples per ray
spacing_fn: Function that dictates sample spacing (ie `lambda x : x` is uniform).
spacing_fn_inv: The inverse of spacing_fn.
train_stratified: Use stratified sampling during training. Defults to True
density_field: Density grid. If provides, samples below weight_threshold as set as invalid.
weight_threshold: Removes samples below threshold weight. Only used if density field is provided.
"""
super().__init__(num_samples=num_samples, density_field=density_field, weight_threshold=weight_threshold)
self.train_stratified = train_stratified
self.spacing_fn = spacing_fn
self.spacing_fn_inv = spacing_fn_inv
@torch.no_grad()
def generate_ray_samples(
self,
ray_bundle: Optional[RayBundle] = None,
num_samples: Optional[int] = None,
) -> RaySamples:
"""Generates position samples accoring to spacing function.
Args:
ray_bundle: Rays to generate samples for
num_samples: Number of samples per ray
Returns:
Positions and deltas for samples along a ray
"""
assert ray_bundle is not None
assert ray_bundle.nears is not None
assert ray_bundle.fars is not None
num_samples = num_samples or self.num_samples
num_rays = ray_bundle.origins.shape[0]
bins = torch.linspace(0.0, 1.0, num_samples + 1).to(ray_bundle.origins.device)[None, ...] # [1, num_samples+1]
s_near, s_far = (self.spacing_fn(x) for x in (ray_bundle.nears, ray_bundle.fars))
bins = self.spacing_fn_inv(bins * s_far + (1 - bins) * s_near) # [num_rays, num_samples+1]
if self.train_stratified and self.training:
t_rand = torch.rand((num_rays, num_samples + 1), dtype=bins.dtype, device=bins.device)
bin_centers = (bins[..., 1:] + bins[..., :-1]) / 2.0
bin_upper = torch.cat([bin_centers, bins[..., -1:]], -1)
bin_lower = torch.cat([bins[..., :1], bin_centers], -1)
bins = bin_lower + (bin_upper - bin_lower) * t_rand
ray_samples = ray_bundle.get_ray_samples(bin_starts=bins[..., :-1, None], bin_ends=bins[..., 1:, None])
return ray_samples
class UniformSampler(SpacedSampler):
"""Sample uniformly along a ray"""
def __init__(
self,
num_samples: int,
train_stratified=True,
density_field: Optional[DensityGrid] = None,
weight_threshold: float = 1e-2,
) -> None:
"""
Args:
num_samples: Number of samples per ray
train_stratified: Use stratified sampling during training. Defults to True
density_field: Density grid. If provides, samples below weight_threshold as set as invalid.
weight_threshold: Removes samples below threshold weight. Only used if density field is provided.
"""
super().__init__(
num_samples=num_samples,
spacing_fn=lambda x: x,
spacing_fn_inv=lambda x: x,
train_stratified=train_stratified,
density_field=density_field,
weight_threshold=weight_threshold,
)
class LinearDisparitySampler(SpacedSampler):
"""Sample linearly in disparity along a ray"""
def __init__(
self,
num_samples: int,
train_stratified=True,
density_field: Optional[DensityGrid] = None,
weight_threshold: float = 1e-2,
) -> None:
"""
Args:
num_samples: Number of samples per ray
train_stratified: Use stratified sampling during training. Defults to True
density_field: Density grid. If provides, samples below weight_threshold as set as invalid.
weight_threshold: Removes samples below threshold weight. Only used if density field is provided.
"""
super().__init__(
num_samples=num_samples,
spacing_fn=lambda x: 1 / x,
spacing_fn_inv=lambda x: 1 / x,
train_stratified=train_stratified,
density_field=density_field,
weight_threshold=weight_threshold,
)
class SqrtSampler(SpacedSampler):
"""Square root sampler along a ray"""
def __init__(
self,
num_samples: int,
train_stratified=True,
density_field: Optional[DensityGrid] = None,
weight_threshold: float = 1e-2,
) -> None:
"""
Args:
num_samples: Number of samples per ray
train_stratified: Use stratified sampling during training. Defults to True
density_field: Density grid. If provides, samples below weight_threshold as set as invalid.
weight_threshold: Removes samples below threshold weight. Only used if density field is provided.
"""
super().__init__(
num_samples=num_samples,
spacing_fn=torch.sqrt,
spacing_fn_inv=lambda x: x**2,
train_stratified=train_stratified,
density_field=density_field,
weight_threshold=weight_threshold,
)
class LogSampler(SpacedSampler):
"""Log sampler along a ray"""
def __init__(
self,
num_samples: int,
train_stratified=True,
density_field: Optional[DensityGrid] = None,
weight_threshold: float = 1e-2,
) -> None:
"""
Args:
num_samples: Number of samples per ray
train_stratified: Use stratified sampling during training. Defults to True
density_field: Density grid. If provides, samples below weight_threshold as set as invalid.
weight_threshold: Removes samples below threshold weight. Only used if density field is provided.
"""
super().__init__(
num_samples=num_samples,
spacing_fn=torch.log,
spacing_fn_inv=torch.exp,
train_stratified=train_stratified,
density_field=density_field,
weight_threshold=weight_threshold,
)
class PDFSampler(Sampler):
"""Sample based on probability distribution"""
def __init__(
self,
num_samples: int,
train_stratified: bool = True,
include_original: bool = True,
density_field: Optional[DensityGrid] = None,
weight_threshold: float = 1e-2,
histogram_padding: float = 0.01,
) -> None:
"""
Args:
num_samples: Number of samples per ray
train_stratified: Randomize location within each bin during training. Defaults to True
include_original: Add original samples to ray. Defaults to True
density_field: Density grid. If provides, samples below weight_threshold as set as invalid.
weight_threshold: Removes samples below threshold weight. Only used if density field is provided.
histogram_padding: Amount to weights prior to computing PDF. Defaults to 0.01.
"""
super().__init__(num_samples=num_samples, density_field=density_field, weight_threshold=weight_threshold)
self.train_stratified = train_stratified
self.include_original = include_original
self.histogram_padding = histogram_padding
@torch.no_grad()
def generate_ray_samples(
self,
ray_bundle: Optional[RayBundle] = None,
ray_samples: Optional[RaySamples] = None,
weights: TensorType[..., "num_samples", 1] = None,
num_samples: Optional[int] = None,
eps: float = 1e-5,
) -> RaySamples:
"""Generates position samples given a distribution.
Args:
ray_bundle: Rays to generate samples for
ray_samples: Existing ray samples
weights: Weights for each bin
num_samples: Number of samples per ray
eps: Small value to prevent numerical issues. Defaults to 1e-5
Returns:
Positions and deltas for samples along a ray
"""
if ray_samples is None or ray_bundle is None:
raise ValueError("ray_samples and ray_bundle must be provided")
num_samples = num_samples or self.num_samples
num_bins = num_samples + 1
weights = weights[..., 0] + self.histogram_padding
# Add small offset to rays with zero weight to prevent NaNs
weights_sum = torch.sum(weights, dim=-1, keepdim=True)
padding = torch.relu(eps - weights_sum)
weights = weights + padding / weights.shape[-1]
weights_sum += padding
pdf = weights / weights_sum
cdf = torch.min(torch.ones_like(pdf), torch.cumsum(pdf, dim=-1))
cdf = torch.cat([torch.zeros_like(cdf[..., :1]), cdf], dim=-1)
if self.train_stratified and self.training:
# Stratified samples between 0 and 1
u = torch.linspace(0.0, 1.0 - (1.0 / num_bins), steps=num_bins, device=cdf.device)
u = u.expand(size=(*cdf.shape[:-1], num_bins))
u = u + torch.rand(size=(*cdf.shape[:-1], num_bins), device=cdf.device) / num_bins
else:
# Uniform samples between 0 and 1
u = torch.linspace(0.0, 1.0, steps=num_bins, device=cdf.device)
u = u.expand(size=(*cdf.shape[:-1], num_bins))
u = u.contiguous()
# Force bins to not have a gap between them. Kinda hacky, should reconsider.
existing_bins = torch.cat(
[
ray_samples.frustums.starts[..., :1, 0],
(ray_samples.frustums.starts[..., 1:, 0] + ray_samples.frustums.ends[..., :-1, 0]) / 2.0,
ray_samples.frustums.ends[..., -1:, 0],
],
dim=-1,
)
inds = torch.searchsorted(cdf, u, side="right")
below = torch.clamp(inds - 1, 0, existing_bins.shape[-1] - 1)
above = torch.clamp(inds, 0, existing_bins.shape[-1] - 1)
cdf_g0 = torch.gather(cdf, -1, below)
bins_g0 = torch.gather(existing_bins, -1, below)
cdf_g1 = torch.gather(cdf, -1, above)
bins_g1 = torch.gather(existing_bins, -1, above)
t = torch.clip(torch.nan_to_num((u - cdf_g0) / (cdf_g1 - cdf_g0), 0), 0, 1)
bins = bins_g0 + t * (bins_g1 - bins_g0)
if self.include_original:
bins, _ = torch.sort(torch.cat([existing_bins, bins], -1), -1)
# Stop gradients
bins = bins.detach()
ray_samples = ray_bundle.get_ray_samples(bin_starts=bins[..., :-1, None], bin_ends=bins[..., 1:, None])
return ray_samples
class NGPSpacedSampler(Sampler):
"""Sampler that matches Instant-NGP paper.
This sampler does ray-box AABB test, ray samples generation and density check, all together.
TODO(ruilongli): check whether fuse AABB test together with `raymarching` can speed things up.
Otherwise we can seperate it out and use collision detecoter that already exists in the repo.
"""
def generate_ray_samples(self) -> RaySamples:
raise RuntimeError("For NGP we fused ray samples and density check together. Please call forward() directly.")
# pylint: disable=arguments-differ
def forward(
self,
ray_bundle: RayBundle,
aabb: TensorType[2, 3],
num_samples: Optional[int] = None,
marching_steps: Optional[int] = 128,
t_min: Optional[TensorType["num_rays"]] = None,
t_max: Optional[TensorType["num_rays"]] = None,
) -> Tuple[RaySamples, TensorType["total_samples", 3], TensorType["total_samples"], TensorType["total_samples"]]:
"""Generate ray samples in a bounding box.
TODO(ruilongli): write a Packed[Ray_samples] class to ray_samples with packed_info.
TODO(ruilongli): maybe move aabb test to the collision detector?
Args:
ray_bundle: Rays to generate samples for
aabb: Bounding box of the scene.
num_samples: Number of samples per ray
Returns:
First return is ray samples in a packed way where only the valid samples are kept.
Second return contains all the information to recover packed samples into unpacked mode for rendering.
The last two returns are t_min and t_max from ray-aabb test.
"""
if self.density_field is None:
raise ValueError("density_field must be set to use NGPSpacedSampler")
num_samples = num_samples or self.num_samples
aabb = aabb.flatten()
rays_o = ray_bundle.origins.contiguous()
rays_d = ray_bundle.directions.contiguous()
scene_scale = (aabb[3:6] - aabb[0:3]).max()
if t_min is None or t_max is None:
# TODO(ruilongli): this clipping here is stupid. Try to avoid that.
t_min, t_max = nerfactory_cuda.ray_aabb_intersect(rays_o, rays_d, aabb)
t_min = torch.clamp(t_min, max=1e10) # type: ignore
t_max = torch.clamp(t_max, max=1e10) # type: ignore
marching_steps = -1 # disable marching mode for now
# TODO(ruilongli): * 16 is for original impl for training. Need to run
# some profiling test with this choice.
max_samples_per_batch = len(rays_o) * num_samples
# marching
packed_info, origins, dirs, starts, ends = nerfactory_cuda.raymarching(
# rays
rays_o,
rays_d,
t_min,
t_max,
# density grid
self.density_field.center,
self.density_field.base_scale,
self.density_field.num_cascades,
self.density_field.resolution,
self.density_field.density_bitfield,
# sampling args
marching_steps,
max_samples_per_batch,
num_samples,
0.0,
scene_scale,
)
# squeeze valid samples
total_samples = max(packed_info[:, -1].sum(), 1)
origins = origins[:total_samples]
dirs = dirs[:total_samples]
starts = starts[:total_samples]
ends = ends[:total_samples]
# return samples
zeros = torch.zeros_like(origins[:, :1])
ray_samples = RaySamples(
frustums=Frustums(origins=origins, directions=dirs, starts=starts, ends=ends, pixel_area=zeros),
)
return ray_samples, packed_info, t_min, t_max
@@ -0,0 +1,109 @@
# Copyright 2022 The Plenoptix Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Ray generator.
"""
import torch
from torch import nn
from torchtyping import TensorType
from nerfactory.cameras.rays import RayBundle
from nerfactory.dataloaders.structs import SceneBounds
class SceneBoundsCollider(nn.Module):
"""Module for setting near and far values for rays."""
def __init__(self, **kwargs) -> None:
self.kwargs = kwargs
super().__init__()
def forward(self, ray_bundle: RayBundle) -> RayBundle:
"""To be implemented."""
raise NotImplementedError
class AABBBoxCollider(SceneBoundsCollider):
"""Module for colliding rays with the scene bounds to compute near and far values."""
def __init__(self, scene_bounds: SceneBounds, **kwargs) -> None:
super().__init__(**kwargs)
self.scene_bounds = scene_bounds
@classmethod
def intersect_with_aabb(
cls, rays_o: TensorType["num_rays", 3], rays_d: TensorType["num_rays", 3], aabb: TensorType[2, 3]
):
"""_summary_
Args:
rays_o (torch.tensor): (num_rays, 3)
rays_d (torch.tensor): (num_rays, 3)
aabb (torch.tensor): (2, 3) This is [min point (x,y,z), max point (x,y,z)]
"""
# avoid divide by zero
dir_fraction = 1.0 / (rays_d + 1e-6)
# x
t1 = (aabb[0, 0] - rays_o[:, 0:1]) * dir_fraction[:, 0:1]
t2 = (aabb[1, 0] - rays_o[:, 0:1]) * dir_fraction[:, 0:1]
# y
t3 = (aabb[0, 1] - rays_o[:, 1:2]) * dir_fraction[:, 1:2]
t4 = (aabb[1, 1] - rays_o[:, 1:2]) * dir_fraction[:, 1:2]
# z
t5 = (aabb[0, 2] - rays_o[:, 2:3]) * dir_fraction[:, 2:3]
t6 = (aabb[1, 2] - rays_o[:, 2:3]) * dir_fraction[:, 2:3]
nears = torch.max(
torch.cat([torch.minimum(t1, t2), torch.minimum(t3, t4), torch.minimum(t5, t6)], dim=1), dim=1
).values
fars = torch.min(
torch.cat([torch.maximum(t1, t2), torch.maximum(t3, t4), torch.maximum(t5, t6)], dim=1), dim=1
).values
# fars < 0: means the ray is behind the camera
# nears > fars: means no intersection
valid_mask = ((fars > nears).float() * (fars > 0).float()).bool()
nears[nears < 0.0] = 0.0
nears[~valid_mask] = 0.0
fars[~valid_mask] = 0.0
return nears, fars, valid_mask
def forward(self, ray_bundle: RayBundle) -> RayBundle:
"""Intersects the rays with the scene bounds and updates the near and far values.
Populates nears and fars fields and returns the ray_bundle.
"""
aabb = self.scene_bounds.aabb
nears, fars, valid_mask = self.intersect_with_aabb(ray_bundle.origins, ray_bundle.directions, aabb)
ray_bundle.nears = nears[..., None]
ray_bundle.fars = fars[..., None]
ray_bundle.valid_mask = valid_mask[..., None]
return ray_bundle
class NearFarCollider(SceneBoundsCollider):
"""Sets the nears and fars with fixed values."""
def __init__(self, near_plane, far_plane, **kwargs) -> None:
self.near_plane = near_plane
self.far_plane = far_plane
super().__init__(**kwargs)
def forward(self, ray_bundle: RayBundle) -> RayBundle:
ones = torch.ones_like(ray_bundle.origins[..., 0:1])
ray_bundle.nears = ones * self.near_plane
ray_bundle.fars = ones * self.far_plane
ray_bundle.valid_mask = ones.bool()
return ray_bundle