diff --git a/pyrad/cameras/rays.py b/pyrad/cameras/rays.py index 7c0597790..bd58118c6 100644 --- a/pyrad/cameras/rays.py +++ b/pyrad/cameras/rays.py @@ -21,13 +21,14 @@ from typing import Optional import torch from torchtyping import TensorType -from pyrad.utils.math import Gaussians, conical_frustum_to_gaussian +from pyrad.utils.math import Gaussians, conical_frustum_to_gaussian from pyrad.utils.misc import is_not_none +from pyrad.utils.tensor_dataclass import TensorDataclass @dataclass -class Frustums: +class Frustums(TensorDataclass): """Describes region of space as a frustum. Args: @@ -83,23 +84,6 @@ class Frustums: pixel_area=torch.ones((1, 1)).to(device), ) - def apply_masks(self, mask: TensorType) -> "Frustums": - """Use valid_mask to mask samples. - - Args: - mask (TensorType): Frustums to keep - - Returns: - Frustum: New set of masked frustums. - """ - return Frustums( - origins=self.origins[mask], - directions=self.directions[mask], - frustum_starts=self.frustum_starts[mask], - frustum_ends=self.frustum_ends[mask], - pixel_area=self.pixel_area[mask], - ) - @dataclass class RaySamples: @@ -114,9 +98,9 @@ class RaySamples: deltas )TensorType[..., 1]): "width" of each sample. """ - frustums: TensorType[..., 3] + frustums: Frustums camera_indices: TensorType[..., 1] = None - valid_mask: TensorType[...] = None + valid_mask: TensorType[..., 1] = None bin_starts: TensorType[..., 1] = None bin_ends: TensorType[..., 1] = None deltas: TensorType[..., 1] = None @@ -162,7 +146,7 @@ class RaySamples: RaySamples: New set of masked samples. """ if is_not_none(self.valid_mask): - frustums = self.frustums.apply_masks(self.valid_mask) + frustums = self.frustums[self.valid_mask] camera_indices = ( self.camera_indices[self.valid_mask] if is_not_none(self.camera_indices) else self.camera_indices ) @@ -180,7 +164,7 @@ class RaySamples: @dataclass -class RayBundle: +class RayBundle(TensorDataclass): """A bundle of ray parameters. Args: @@ -188,18 +172,18 @@ class RayBundle: directions (TensorType[..., 3]): Unit ray direction vector pixel_area (TensorType[..., 1]): Projected area of pixel a distance 1 away from origin. camera_indices (TensorType[..., 1]): Camera indices - nears (TensorType[...]): Distance along ray to start sampling - fars (TensorType[...]): Rays Distance along ray to stop sampling - valid_mask (TensorType[...]): Rays that are valid + nears (TensorType[..., 1]): Distance along ray to start sampling + fars (TensorType[..., 1]): Rays Distance along ray to stop sampling + valid_mask (TensorType[..., 1]): Rays that are valid """ origins: TensorType["num_rays", 3] directions: TensorType["num_rays", 3] pixel_area: TensorType["num_rays", 1] camera_indices: Optional[TensorType["num_rays", 1]] = None - nears: Optional[TensorType["num_rays"]] = None - fars: Optional[TensorType["num_rays"]] = None - valid_mask: Optional[TensorType["num_rays"]] = None + nears: Optional[TensorType["num_rays", 1]] = None + fars: Optional[TensorType["num_rays", 1]] = None + valid_mask: Optional[TensorType["num_rays", 1]] = None num_rays_per_chunk: int = None def move_to_device(self, device: torch.device) -> None: @@ -219,7 +203,7 @@ class RayBundle: Args: camera_index (int): Camera index. """ - self.camera_indices = torch.ones_like(self.origins[..., 0]).long() * camera_index + self.camera_indices = torch.ones_like(self.origins[..., 0:1]).long() * camera_index def __len__(self): num_rays = torch.numel(self.origins) // self.origins.shape[-1] @@ -236,31 +220,7 @@ class RayBundle: """ assert num_rays <= len(self) indices = random.sample(range(len(self)), k=num_rays) - return RayBundle( - origins=self.origins[indices], - directions=self.directions[indices], - pixel_area=self.pixel_area[indices], - camera_indices=self.camera_indices[indices], - ) - - def get_masked_ray_bundle(self, valid_mask: TensorType) -> "RayBundle": - """Return a masked instance of the ray bundle. - - Args: - valid_mask (TensorType): Mask of rays to keep. - - Returns: - RayBundle: RayBundle of masked rays. - """ - return RayBundle( - origins=self.origins[valid_mask], - directions=self.directions[valid_mask], - pixel_area=self.pixel_area[valid_mask], - camera_indices=self.camera_indices[valid_mask] if is_not_none(self.camera_indices) else None, - nears=self.nears[valid_mask] if is_not_none(self.nears) else None, - fars=self.fars[valid_mask] if is_not_none(self.fars) else None, - valid_mask=self.valid_mask[valid_mask] if is_not_none(self.valid_mask) else None, - ) + return self[indices] def get_row_major_sliced_ray_bundle(self, start_idx: int, end_idx: int) -> "RayBundle": """Flattens RayBundle and extracts chunk given start and end indicies. @@ -273,15 +233,7 @@ class RayBundle: RayBundle: Flattened RayBundle with end_idx-start_idx rays. """ - camera_indices = ( - self.camera_indices.view(-1)[start_idx:end_idx] if not isinstance(self.camera_indices, type(None)) else None - ) - return RayBundle( - origins=self.origins.view(-1, 3)[start_idx:end_idx], - directions=self.directions.view(-1, 3)[start_idx:end_idx], - pixel_area=self.pixel_area.view(-1, 1)[start_idx:end_idx], - camera_indices=camera_indices, - ) + return self.flatten()[start_idx:end_idx] def get_ray_samples( self, bin_starts: TensorType["num_rays", "num_samples"], bin_ends: TensorType["num_rays", "num_samples"] @@ -305,17 +257,17 @@ class RayBundle: deltas = dists * torch.norm(self.directions[:, None, :], dim=-1) if is_not_none(self.camera_indices): - camera_indices = self.camera_indices.unsqueeze(1).repeat(1, num_samples) + camera_indices = self.camera_indices.repeat(1, num_samples) else: camera_indices = None frustums = Frustums( - origins=self.origins.unsqueeze(1).repeat(1, num_samples, 1), - directions=self.directions.unsqueeze(1).repeat(1, num_samples, 1), - frustum_starts=bin_starts[:, :, None].to(device), - frustum_ends=bin_ends[:, :, None].to(device), - pixel_area=self.pixel_area.unsqueeze(1).repeat(1, num_samples, 1).to(device), - ) + origins=self.origins[:, None, :], + directions=self.directions[:, None, :], + frustum_starts=bin_starts[:, :, None], + frustum_ends=bin_ends[:, :, None], + pixel_area=self.pixel_area[:, None, :], + ).to(device) ray_samples = RaySamples( frustums=frustums, diff --git a/pyrad/fields/base.py b/pyrad/fields/base.py index 8ed8ec4f8..1ba11cb2c 100644 --- a/pyrad/fields/base.py +++ b/pyrad/fields/base.py @@ -82,7 +82,7 @@ class Field(nn.Module): if is_not_none(valid_mask): # Hacky handling of empty masks. Tests on a single ray but doesn't use results if not valid_mask.any(): - ray_samples = RaySamples(frustums=Frustums.get_mock_frustum(device=valid_mask.device)) + ray_samples = RaySamples(frustums=Frustums.get_mock_frustum().to(valid_mask.device)) else: ray_samples = ray_samples.apply_masks() density_masked, density_embedding_masked = self.get_density(ray_samples) diff --git a/pyrad/graphs/base.py b/pyrad/graphs/base.py index 33df56c7e..ed133d0df 100644 --- a/pyrad/graphs/base.py +++ b/pyrad/graphs/base.py @@ -143,8 +143,8 @@ class Graph(AbstractGraph): return outputs # during training, keep only the rays that intersect the scene. discard the rest - valid_mask = intersected_ray_bundle.valid_mask - masked_intersected_ray_bundle = intersected_ray_bundle.get_masked_ray_bundle(valid_mask) + valid_mask = intersected_ray_bundle.valid_mask[..., 0] + masked_intersected_ray_bundle = intersected_ray_bundle[valid_mask] masked_batch = get_masked_dict(batch, valid_mask) # NOTE(ethan): this is really slow if on CPU! outputs = self.get_outputs(masked_intersected_ray_bundle) loss_dict = self.get_loss_dict(outputs=outputs, batch=masked_batch) diff --git a/pyrad/graphs/modules/ray_generator.py b/pyrad/graphs/modules/ray_generator.py index c057fe35c..db2d8c17c 100644 --- a/pyrad/graphs/modules/ray_generator.py +++ b/pyrad/graphs/modules/ray_generator.py @@ -65,5 +65,5 @@ class RayGenerator(nn.Module): ray_bundle = self.camera_class.generate_rays( intrinsics=intrinsics, camera_to_world=camera_to_world, coords=coords ) - ray_bundle.camera_indices = c + ray_bundle.camera_indices = c[..., None] # ["num_rays",1] return ray_bundle diff --git a/pyrad/graphs/modules/ray_sampler.py b/pyrad/graphs/modules/ray_sampler.py index 3484e3402..5752060eb 100644 --- a/pyrad/graphs/modules/ray_sampler.py +++ b/pyrad/graphs/modules/ray_sampler.py @@ -105,7 +105,7 @@ class SpacedSampler(Sampler): bins = torch.linspace(0.0, 1.0, num_samples + 1).to(ray_bundle.origins.device)[None, ...] # [1, num_samples+1] - s_near, s_far = [self.spacing_fn(x) for x in (ray_bundle.nears[:, None], ray_bundle.fars[:, None])] + s_near, s_far = [self.spacing_fn(x) for x in (ray_bundle.nears, ray_bundle.fars)] bins = self.spacing_fn_inv(bins * s_far + (1 - bins) * s_near) # [num_rays, num_samples+1] if self.train_stratified and self.training: diff --git a/pyrad/graphs/modules/scene_colliders.py b/pyrad/graphs/modules/scene_colliders.py index 65c9792b8..32e1bffd8 100644 --- a/pyrad/graphs/modules/scene_colliders.py +++ b/pyrad/graphs/modules/scene_colliders.py @@ -86,9 +86,9 @@ class AABBBoxCollider(SceneBoundsCollider): """ aabb = self.scene_bounds.aabb nears, fars, valid_mask = self.intersect_with_aabb(ray_bundle.origins, ray_bundle.directions, aabb) - ray_bundle.nears = nears - ray_bundle.fars = fars - ray_bundle.valid_mask = valid_mask + ray_bundle.nears = nears[..., None] + ray_bundle.fars = fars[..., None] + ray_bundle.valid_mask = valid_mask[..., None] return ray_bundle @@ -101,7 +101,7 @@ class NearFarCollider(SceneBoundsCollider): super().__init__(**kwargs) def forward(self, ray_bundle: RayBundle) -> RayBundle: - ones = torch.ones_like(ray_bundle.origins[:, 0]) + ones = torch.ones_like(ray_bundle.origins[..., 0:1]) ray_bundle.nears = ones * self.near_plane ray_bundle.fars = ones * self.far_plane ray_bundle.valid_mask = ones.bool() diff --git a/pyrad/utils/tensor_dataclass.py b/pyrad/utils/tensor_dataclass.py index 3afbab2ba..3fe57762c 100644 --- a/pyrad/utils/tensor_dataclass.py +++ b/pyrad/utils/tensor_dataclass.py @@ -15,14 +15,15 @@ """Tensor dataclass""" import dataclasses -from typing import Tuple +from typing import Tuple, Union import numpy as np import torch class TensorDataclass: - """Data class of tensor with the same size batch. Allows indexing and standard tensor ops. + """Data class of tensors with the same size batch. Allows indexing and standard tensor ops. + Fields that are not Tensors will not be batched unless they are also a TensorDataclass. Example: @@ -54,22 +55,40 @@ class TensorDataclass: if not dataclasses.is_dataclass(self): raise TypeError("TensorDataclass must be a dataclass") - fields = dataclasses.asdict(self) - batch_shapes = [v.shape[:-1] for v in fields.values() if v is not None] + field_names = [f.name for f in dataclasses.fields(self)] + batch_shapes = [] + for f in field_names: + v = self.__getattribute__(f) + if v is not None: + if isinstance(v, torch.Tensor): + batch_shapes.append(v.shape[:-1]) + elif isinstance(v, TensorDataclass): + batch_shapes.append(v.shape) if len(batch_shapes) == 0: raise ValueError("TensorDataclass must have at least one tensor") batch_shape = torch.broadcast_shapes(*batch_shapes) - for k, v in fields.items(): + for f in field_names: + v = self.__getattribute__(f) if v is not None: - self.__setattr__(k, torch.broadcast_to(v, (*batch_shape, v.shape[-1]))) + if isinstance(v, torch.Tensor): + self.__setattr__(f, v.broadcast_to((*batch_shape, v.shape[-1]))) + elif isinstance(v, TensorDataclass): + self.__setattr__(f, v.broadcast_to(batch_shape)) self.__setattr__("_shape", batch_shape) def __getitem__(self, indices) -> "TensorDataclass": - if isinstance(indices, int): + if isinstance(indices, torch.Tensor): + return self._apply_fn_to_fields(lambda x: x[indices]) + if isinstance(indices, (int, slice)): indices = (indices,) - return self._apply_fn_to_fields(lambda x: x[indices + (slice(None),)]) + tensor_fn = lambda x: x[indices + (slice(None),)] + dataclass_fn = lambda x: x[indices] + return self._apply_fn_to_fields(tensor_fn, dataclass_fn) + + def __setitem__(self, indices, value) -> "TensorDataclass": + raise RuntimeError("Index assignment is not supported for TensorDataclass") def __len__(self) -> int: return self.shape[0] @@ -110,7 +129,9 @@ class TensorDataclass: """ if isinstance(shape, int): shape = (shape,) - return self._apply_fn_to_fields(lambda x: x.reshape((*shape, x.shape[-1]))) + tensor_fn = lambda x: x.reshape((*shape, x.shape[-1])) + dataclass_fn = lambda x: x.reshape(shape) + return self._apply_fn_to_fields(tensor_fn, dataclass_fn) def flatten(self) -> "TensorDataclass": """Returns a new TensorDataclass with flattened batch dimensions @@ -120,7 +141,7 @@ class TensorDataclass: """ return self.reshape((-1,)) - def broadcast_to(self, shape: Tuple[int]) -> "TensorDataclass": + def broadcast_to(self, shape: Union[torch.Size, Tuple[int]]) -> "TensorDataclass": """Returns a new TensorDataclass broadcast to new shape. Args: @@ -129,20 +150,38 @@ class TensorDataclass: Returns: TensorDataclass: A new TensorDataclass with the same data but with a new shape. """ - return self._apply_fn_to_fields(lambda x: x.broadcast_to((*shape, x.shape[-1]))) - def _apply_fn_to_fields(self, fn: callable) -> "TensorDataclass": + def to(self, device) -> "TensorDataclass": + """Returns a new TensorDataclass with the same data but on the specified device. + + Args: + device: The device to place the tensor dataclass. + + Returns: + TensorDataclass: A new TensorDataclass with the same data but on the specified device. + """ + return self._apply_fn_to_fields(lambda x: x.to(device)) + + def _apply_fn_to_fields(self, fn: callable, dataclass_fn: callable = None) -> "TensorDataclass": """Applies a function to all fields of the tensor dataclass. Args: - fn (callable): The function to apply to all fields. + fn (callable): The function to apply to tensor fields. + dataclass_fn (callable): The function to apply to TensorDataclass fields. Else use fn. Returns: TensorDataclass: A new TensorDataclass with the same data but with a new shape. """ - fields = dataclasses.asdict(self) - fields = {k: fn(v) for k, v in fields.items() if v is not None} + field_names = [f.name for f in dataclasses.fields(self)] + new_fields = {} + for f in field_names: + v = self.__getattribute__(f) + if v is not None: + if isinstance(v, TensorDataclass) and dataclass_fn is not None: + new_fields[f] = dataclass_fn(v) + elif isinstance(v, (torch.Tensor, TensorDataclass)): + new_fields[f] = fn(v) - return dataclasses.replace(self, **fields) + return dataclasses.replace(self, **new_fields) diff --git a/tests/cameras/test_rays.py b/tests/cameras/test_rays.py index 3b3d70c16..e2a75e02b 100644 --- a/tests/cameras/test_rays.py +++ b/tests/cameras/test_rays.py @@ -56,7 +56,7 @@ def test_frustum_apply_masks(): ) mask = torch.tensor([False, True, False, True, True], dtype=torch.bool) - frustum = frustum.apply_masks(mask) + frustum = frustum[mask] assert frustum.origins.shape == (3, 3) assert frustum.directions.shape == (3, 3) diff --git a/tests/utils/test_tensor_dataclass.py b/tests/utils/test_tensor_dataclass.py index e2dcff02d..63814955f 100644 --- a/tests/utils/test_tensor_dataclass.py +++ b/tests/utils/test_tensor_dataclass.py @@ -2,20 +2,26 @@ Test tensor dataclass """ from dataclasses import dataclass -from typing import Any import pytest import torch from pyrad.utils.tensor_dataclass import TensorDataclass +@dataclass +class TestNestedClass(TensorDataclass): + """Dummy dataclass""" + + x: torch.Tensor + + @dataclass class TestTensorDataclass(TensorDataclass): """Dummy dataclass""" a: torch.Tensor b: torch.Tensor - c: torch.Tensor = None + c: TestNestedClass = None def test_init(): @@ -25,7 +31,7 @@ def test_init(): class Dummy(TensorDataclass): """Dummy dataclass""" - dummy_vals: Any = None + dummy_vals: torch.Tensor = None dummy = Dummy(dummy_vals=torch.ones(1)) with pytest.raises(ValueError): @@ -53,7 +59,7 @@ def test_broadcasting(): def test_tensor_ops(): - """Test get shape""" + """Test tensor operations""" a = torch.ones((4, 6, 3)) b = torch.ones((6, 2)) @@ -75,10 +81,13 @@ def test_tensor_ops(): assert flattened.shape == (24,) assert flattened.a.shape == (24, 3) assert flattened.b.shape == (24, 2) + assert flattened[0:4].shape == (4,) # Test indexing operations assert tensor_dataclass[:, 1].shape == (4,) assert tensor_dataclass[:, 1].a.shape == (4, 3) + assert tensor_dataclass[:, 0:2].shape == (4, 2) + assert tensor_dataclass[:, 0:2].a.shape == (4, 2, 3) assert tensor_dataclass[..., 1].shape == (4,) assert tensor_dataclass[..., 1].a.shape == (4, 3) assert tensor_dataclass[0].shape == (6,) @@ -94,6 +103,42 @@ def test_tensor_ops(): assert tensor_dataclass[..., 0].shape == (2, 3) assert tensor_dataclass[..., 0].a.shape == (2, 3, 5) + mask = torch.rand(size=(2, 3)) > 0.5 + assert tensor_dataclass[mask].ndim == 2 + + +def test_nested_class(): + """Test nested TensorDataclasses""" + + a = torch.ones((4, 6, 3)) + b = torch.ones((6, 2)) + c = TestNestedClass(x=torch.ones(6, 5)) + tensor_dataclass = TestTensorDataclass(a=a, b=b, c=c) + + assert tensor_dataclass.shape == (4, 6) + assert tensor_dataclass.a.shape == (4, 6, 3) + assert tensor_dataclass.b.shape == (4, 6, 2) + assert tensor_dataclass.c.shape == (4, 6) + assert tensor_dataclass.c.x.shape == (4, 6, 5) + assert tensor_dataclass.size == 24 + assert tensor_dataclass.c.size == 24 + + reshaped = tensor_dataclass.reshape((2, 12)) + assert reshaped.shape == (2, 12) + assert reshaped.c.shape == (2, 12) + assert reshaped.c.x.shape == (2, 12, 5) + + flattened = tensor_dataclass.flatten() + assert flattened.c.shape == (24,) + assert flattened.c.x.shape == (24, 5) + + # Test indexing operations + assert tensor_dataclass[:, 1].c.shape == (4,) + assert tensor_dataclass[:, 1].c.x.shape == (4, 5) + + mask = torch.rand(size=(4,)) > 0.5 + assert tensor_dataclass[mask].c.ndim == 2 + def test_iter(): """Test iterating over tensor dataclass""" @@ -109,3 +154,4 @@ if __name__ == "__main__": test_broadcasting() test_tensor_ops() test_iter() + test_nested_class()