""" Camera Models """ from abc import abstractmethod from dataclasses import dataclass from typing import Optional, Type import torch from torch.nn.functional import normalize from torchtyping import TensorType from pyrad.structures.rays import RayBundle @dataclass class Rays: """Camera rays. Can be arbitrary dimension""" origin: TensorType[..., 3] direction: TensorType[..., 3] class Camera: """Base Camera. Intended to be subclassed""" def __init__(self, camera_to_world: Optional[TensorType[3, 4]] = torch.eye(4)[:3]) -> None: self.camera_to_world = camera_to_world @abstractmethod def get_num_intrinsics_params(self) -> int: """ Returns: int: number of optimizable intrinsic parameters """ return @abstractmethod def get_intrinsics(self) -> torch.Tensor: """ Returns: torch.Tensor: Intrinsics matrix """ return def get_camera_to_world(self) -> TensorType[3, 4]: """ Returns: int: Camera to world transformation """ return self.camera_to_world @abstractmethod def get_image_height(self) -> int: """ Returns: int: Image height """ return @abstractmethod def get_image_width(self) -> int: """ Returns: int: Image width """ return def get_image_coords(self, pixel_offset: float = 0.5) -> TensorType["image_height", "image_width", 2]: """ Args: pixel_offset (float): Offset for each pixel. Defaults to center of pixel (0.5) Returns: TensorType["image_height", "image_width", 2]: Grid of image coordinates. """ image_height = self.get_image_height() image_width = self.get_image_width() image_coords = torch.meshgrid(torch.arange(image_height), torch.arange(image_width), indexing="ij") image_coords = torch.stack(image_coords, dim=-1) + pixel_offset # stored as (y, x) coordinates return image_coords @classmethod @abstractmethod def generate_rays( cls, intrinsics: TensorType[..., "num_intrinsics_params"], camera_to_world: TensorType[..., 3, 4], coords: TensorType[..., 2], ) -> RayBundle: """ Args: intrinsics (TensorType[..., "num_intrinsics_params"]): Camera intrinsics camera_to_world (TensorType[..., 3, 4]): Camera to world transformation matrix coords (TensorType[..., 2]): Image grid coordinates Returns: RayBundle: A bundle of rays for each grid coordinate. """ return def generate_camera_rays(self, device=None) -> RayBundle: """Generate rays for the camera. Returns: Rays: Camera rays of shape [height, width] """ if device is None: device = self.camera_to_world.device height = self.get_image_height() width = self.get_image_width() intrinsics = ( self.get_intrinsics().unsqueeze(0).repeat(height, width, 1).to(device) ) # (num_rays, num_intrinsics_params) camera_to_world = self.camera_to_world.unsqueeze(0).repeat(height, width, 1, 1).to(device) # (num_rays, 3, 4) coords = self.get_image_coords().to(device) ray_bundle = self.generate_rays(intrinsics, camera_to_world, coords) return ray_bundle class PinholeCamera(Camera): """Pinhole camera model.""" def __init__( self, cx: float, cy: float, fx: float, fy: float, camera_to_world: Optional[TensorType[3, 4]] = torch.eye(4)[:3] ): super().__init__(camera_to_world) self.cx = cx self.cy = cy self.fx = fx self.fy = fy def get_num_intrinsics_params(self) -> int: return 4 def get_image_width(self) -> int: return int(self.cx * 2.0) def get_image_height(self) -> int: return int(self.cy * 2.0) def get_intrinsics_matrix(self) -> TensorType[3, 3]: """ Returns: TensorType[3, 3]: Pinhole camera intrinsics matrix """ K = torch.tensor( [[self.fx, 0.0, self.cx], [0.0, self.fy, self.cy], [0, 0, 1.0]], dtype=torch.float32, ) return K def get_intrinsics(self) -> torch.Tensor: return torch.tensor([self.cx, self.cy, self.fx, self.fy]) @classmethod def fx_index(cls): """TODO(ethan): redo this in a better way. Ideally we can dynamically grab the focal length parameters depending on Simple vs. not Simple Pinhole Model. Returns: _type_: _description_ """ return 2 @classmethod def fy_index(cls): """_summary_ Returns: _type_: _description_ """ return 3 @classmethod def generate_rays( cls, intrinsics: TensorType[..., "num_intrinsics_params"], camera_to_world: TensorType[..., 3, 4], coords: TensorType[..., 2], ) -> RayBundle: cx = intrinsics[..., 0:1] cy = intrinsics[..., 1:2] fx = intrinsics[..., cls.fx_index() : cls.fx_index() + 1] fy = intrinsics[..., cls.fy_index() : cls.fy_index() + 1] y = coords[..., 0:1] # (..., 1) x = coords[..., 1:2] # (..., 1) original_directions = torch.cat([(x - cx) / fx, -(y - cy) / fy, -torch.ones_like(x)], -1) # (..., 3) rotation = camera_to_world[..., :3, :3] # (..., 3, 3) directions = torch.sum( original_directions[..., None, :] * rotation, dim=-1 ) # (..., 1, 3) * (..., 3, 3) -> (..., 3) directions = normalize(directions, dim=-1) origins = camera_to_world[..., :3, 3] # (..., 3) ## Calculate pixel area directly dirx_min = normalize(torch.cat([(x - cx - 0.5) / fx, -(y - cy) / fy, -torch.ones_like(x)], -1), dim=-1) dirx_max = normalize(torch.cat([(x - cx + 0.5) / fx, -(y - cy) / fy, -torch.ones_like(x)], -1), dim=-1) diry_min = normalize(torch.cat([(x - cx) / fx, -(y - cy - 0.5) / fy, -torch.ones_like(x)], -1), dim=-1) diry_max = normalize(torch.cat([(x - cx) / fx, -(y - cy + 0.5) / fy, -torch.ones_like(x)], -1), dim=-1) dx = torch.sqrt(torch.sum((dirx_max - dirx_min) ** 2, dim=-1)) dy = torch.sqrt(torch.sum((diry_max - diry_min) ** 2, dim=-1)) pixel_area = dx * dy return RayBundle(origins=origins, directions=directions, pixel_area=pixel_area[..., None]) class SimplePinholeCamera(PinholeCamera): """Simple Pinhole Camera model.""" def __init__(self, cx: float, cy: float, f: float, camera_to_world: Optional[TensorType[3, 4]] = torch.eye(4)[:3]): super().__init__(cx, cy, f, f, camera_to_world) @classmethod def fx_index(cls): return 2 @classmethod def fy_index(cls): return 2 class EquirectangularCamera(Camera): """Equirectangular (360 degree) camera model.""" def __init__(self, height: int, width: int, camera_to_world: Optional[TensorType[3, 4]] = torch.eye(4)[:3]): super().__init__(camera_to_world) self.height = height self.width = width def get_num_intrinsics_params(self): return 2 def get_image_width(self): return self.width def get_image_height(self): return self.height def get_intrinsics(self) -> torch.Tensor: return torch.tensor([self.height, self.width]) @classmethod def generate_rays( cls, intrinsics: TensorType[..., 2], camera_to_world: TensorType[..., 3, 4], coords: TensorType[..., 2], ) -> RayBundle: y = coords[..., 0:1] # (num_rays, 1) x = coords[..., 1:2] # (num_rays, 1) height = intrinsics[..., 0:1] width = intrinsics[..., 1:2] phi = y / height * torch.pi theta = -x / width * 2 * torch.pi d_phi_min = (y + 0.5) / height * torch.pi d_phi_max = (y - 0.5) / height * torch.pi d_theta_min = -(x - 0.5) / width * 2 * torch.pi d_theta_max = -(x + 0.5) / width * 2 * torch.pi s_phi = torch.sin(phi) c_phi = torch.cos(phi) s_theta = torch.sin(theta) c_theta = torch.cos(theta) directions = torch.cat([c_theta * s_phi, s_theta * s_phi, c_phi], dim=-1) origins = camera_to_world[..., :3, 3] # (num_rays, 3) ## Calculate area directly dirx_min = normalize( torch.cat([torch.cos(d_theta_min) * s_phi, torch.sin(d_theta_min) * s_phi, c_phi], -1), dim=-1 ) dirx_max = normalize( torch.cat([torch.cos(d_theta_max) * s_phi, torch.sin(d_theta_max) * s_phi, c_phi], -1), dim=-1 ) diry_min = normalize( torch.cat([c_theta * torch.sin(d_phi_min), s_theta * torch.sin(d_phi_min), torch.cos(d_phi_min)], -1), dim=-1, ) diry_max = normalize( torch.cat([c_theta * torch.sin(d_phi_max), s_theta * torch.sin(d_phi_max), torch.cos(d_phi_max)], -1), dim=-1, ) dx = torch.sqrt(torch.sum((dirx_max - dirx_min) ** 2, dim=-1)) dy = torch.sqrt(torch.sum((diry_max - diry_min) ** 2, dim=-1)) pixel_area = dx * dy return RayBundle(origins=origins, directions=directions, pixel_area=pixel_area) def get_camera_model(num_intrinsics_params: int) -> Type[Camera]: """Returns the camera model given the specified number of intrinsics parameters. Args: num_intrinsics_params (int): Number of intrinsic parametes. Returns: Camera: Camera model """ if num_intrinsics_params == 2: return EquirectangularCamera if num_intrinsics_params == 3: return SimplePinholeCamera if num_intrinsics_params == 4: return PinholeCamera raise NotImplementedError