From 04d0165a247916c3cfe48d1c4ce8e0b3f8d6a6fb Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Mon, 29 Aug 2022 21:09:02 +0100 Subject: [PATCH] Camera Refactor (#301) * Start camera refactor * Training runs, but transforms incorrect * Cameras render correctly in viewer. * Fix eval * cleanup * cleanup --- .vscode/launch.json | 22 +- environment/requirements.txt | 1 + nerfactory/cameras/camera_paths.py | 66 +-- nerfactory/cameras/cameras.py | 569 ++++++--------------- nerfactory/cameras/utils.py | 30 +- nerfactory/dataloaders/base.py | 11 +- nerfactory/dataloaders/datasets.py | 78 ++- nerfactory/dataloaders/eval.py | 38 +- nerfactory/dataloaders/structs.py | 17 +- nerfactory/models/base.py | 6 - nerfactory/models/modules/ray_generator.py | 28 +- nerfactory/utils/plotly.py | 18 +- nerfactory/viewer/server/viewer_utils.py | 20 +- scripts/run_eval.py | 17 +- tests/cameras/test_cameras.py | 27 +- 15 files changed, 334 insertions(+), 614 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index a2cef86e5..6af3fad15 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -24,7 +24,8 @@ "type": "python", "request": "launch", "program": "scripts/run_train.py", - "console": "integratedTerminal" + "console": "integratedTerminal", + "args": ["vanilla_nerf"] }, { "name": "Python: run_train (cpu)", @@ -32,9 +33,7 @@ "request": "launch", "program": "scripts/run_train.py", "console": "integratedTerminal", - "args": [ - "machine.num_gpus=0" - ] + "args": ["machine.num_gpus=0"] }, { "name": "Python: run_train ngp (gpu)", @@ -42,9 +41,7 @@ "request": "launch", "program": "scripts/run_train.py", "console": "integratedTerminal", - "args": [ - "instant_ngp" - ] + "args": ["instant_ngp"] }, { "name": "Python: run_train mipnerf 360 (gpu)", @@ -52,9 +49,7 @@ "request": "launch", "program": "scripts/run_train.py", "console": "integratedTerminal", - "args": [ - "mipnerf_360" - ] + "args": ["mipnerf_360"] }, { "name": "Python: run_train nerfw (gpu)", @@ -73,10 +68,7 @@ "request": "launch", "program": "scripts/run_train.py", "console": "integratedTerminal", - "args": [ - "semantic_nerf", - "++data.use_preprocessing_cache=true" - ] + "args": ["semantic_nerf", "++data.use_preprocessing_cache=true"] } ] -} \ No newline at end of file +} diff --git a/environment/requirements.txt b/environment/requirements.txt index 80411cbb0..3cef67ffb 100644 --- a/environment/requirements.txt +++ b/environment/requirements.txt @@ -16,6 +16,7 @@ matplotlib mediapy msgpack msgpack_numpy +numpy-quaternion opencv-python plotly==5.7.0 protobuf==3.20.0 diff --git a/nerfactory/cameras/camera_paths.py b/nerfactory/cameras/camera_paths.py index 77f6eb9ea..776680325 100644 --- a/nerfactory/cameras/camera_paths.py +++ b/nerfactory/cameras/camera_paths.py @@ -16,30 +16,16 @@ Code for camera paths. """ -import copy -from dataclasses import dataclass -from typing import List, Optional, Tuple +from typing import Optional, Tuple import torch import nerfactory.cameras.utils as camera_utils -from nerfactory.cameras.cameras import ( - Camera, - PinholeCamera, - get_camera, - get_intrinsics_from_intrinsics_matrix, -) +from nerfactory.cameras.cameras import Cameras from nerfactory.cameras.utils import get_interpolated_poses_many -@dataclass -class CameraPath: - """A camera path.""" - - cameras: List[Camera] - - -def get_interpolated_camera_path(camera_a: PinholeCamera, camera_b: PinholeCamera, steps: int) -> CameraPath: +def get_interpolated_camera_path(cameras: Cameras, steps: int) -> Cameras: """Generate a camera path between two cameras. Args: @@ -48,38 +34,24 @@ def get_interpolated_camera_path(camera_a: PinholeCamera, camera_b: PinholeCamer steps: The number of steps to interpolate between the two cameras. Returns: - CameraPath: A camera path. + A new set of cameras along a path. """ - device = camera_a.device - Ka = camera_a.get_intrinsics_matrix().cpu().numpy() - pose_a = camera_a.get_camera_to_world_h().cpu().numpy() - Kb = camera_b.get_intrinsics_matrix().cpu().numpy() - pose_b = camera_b.get_camera_to_world_h().cpu().numpy() - poses = [pose_a, pose_b] - Ks = [Ka, Kb] + Ks = cameras.get_intrinsics_matrices().cpu().numpy() + poses = cameras.camera_to_worlds().cpu().numpy() poses, Ks = get_interpolated_poses_many(poses, Ks, steps_per_transition=steps) - cameras = [] - for pose, K in zip(poses, Ks): - intrinsics = get_intrinsics_from_intrinsics_matrix(K).to(device).float() - # TODO: this makes cx and cy an integer, but this code should be fixed - # it was added to avoid floating point errors when rescaling and the image resolution - # being different per rendered image in a camera path - intrinsics[:2] = torch.round(intrinsics[:2]) - camera_to_world = torch.from_numpy(pose[:3]).to(device).float() - camera = get_camera(intrinsics, camera_to_world, camera_index=camera_a.camera_index) - cameras.append(camera) - return CameraPath(cameras=cameras) + cameras = Cameras(fx=Ks[:, 0, 0], fy=Ks[:, 1, 1], cx=Ks[0, 0, 2], cy=Ks[0, 1, 2], camera_to_worlds=poses) + return cameras def get_spiral_path( - camera: PinholeCamera, + camera: Cameras, steps: int = 30, radius: Optional[float] = None, radiuses: Optional[Tuple[float]] = None, rots: int = 2, zrate: float = 0.5, -) -> CameraPath: +) -> Cameras: """ Returns a list of camera in a sprial trajectory. @@ -92,7 +64,7 @@ def get_spiral_path( zrate: How much to change the z position of the camera. Returns: - CameraPath: A spiral camera path. + A spiral camera path. """ assert radius is not None or radiuses is not None, "Either radius or radiuses must be specified." @@ -104,12 +76,14 @@ def get_spiral_path( raise ValueError("Only one of radius or radiuses must be specified.") # TODO: don't hardcode this. pass this in - up = camera.camera_to_world[:3, 2] # scene is z up + up = camera.camera_to_worlds[0, :3, 2] # scene is z up # up = camera.camera_to_world[:3, 1] # this will rotate 90 degrees - focal = min(camera.fx, camera.fy) + focal = torch.min(camera.fx[0], camera.fy[0]) target = torch.tensor([0, 0, -focal], device=camera.device) # camera looking in -z direction - c2wh_global = camera.get_camera_to_world_h() + c2w = camera.camera_to_worlds[0] + ones = torch.tensor([0, 0, 0, 1], device=c2w.device)[None] + c2wh_global = torch.cat([c2w, ones], dim=0) local_c2whs = [] for theta in torch.linspace(0.0, 2.0 * torch.pi * rots, steps + 1)[:-1]: @@ -122,11 +96,9 @@ def get_spiral_path( c2wh = torch.cat([c2w, ones], dim=0) local_c2whs.append(c2wh) - cameras = [] + new_c2ws = [] for local_c2wh in local_c2whs: - cam = copy.deepcopy(camera) c2wh = torch.matmul(c2wh_global, local_c2wh) - cam.camera_to_world = c2wh[:3, :4] - cameras.append(cam) + new_c2ws.append(c2wh[:3, :4]) - return CameraPath(cameras=cameras) + return Cameras(fx=camera.fx[0], fy=camera.fy[0], cx=camera.cx, cy=camera.cy, camera_to_worlds=new_c2ws) diff --git a/nerfactory/cameras/cameras.py b/nerfactory/cameras/cameras.py index c35773129..9455d2c74 100644 --- a/nerfactory/cameras/cameras.py +++ b/nerfactory/cameras/cameras.py @@ -16,8 +16,8 @@ Camera Models """ import base64 -from abc import abstractmethod -from typing import Dict, Optional, Tuple, Type +from enum import Enum, auto +from typing import Dict, Optional, Tuple, Union import cv2 import numpy as np @@ -28,87 +28,96 @@ from torchtyping import TensorType from nerfactory.cameras.rays import RayBundle -class Camera: - """Base Camera. Intended to be subclassed. +class CameraType(Enum): + """Supported camera types.""" + + PERSPECTIVE = auto + FISHEYE = auto + + +class Cameras: + """Dataset inputs for the image dataset and the ray generator. + + Note: currently only supports cameras with the same principal points and types. Args: - camera_to_world: Camera to world transformation - camera_index: Camera index + camera_to_worlds: Tensor of per-image c2w matrices, in [R | t] format. + fx: Focal length x. + fy: Focal length y. + cx: Principal point x. + cy: Principal point y. + distortion_params: OpenCV 6 radial distortion coefficients. + camera_type: Type of camera model. """ def __init__( - self, camera_to_world: TensorType[3, 4] = torch.eye(4)[:3], camera_index: Optional[int] = None - ) -> None: - self.camera_to_world = camera_to_world - self.camera_index = camera_index + self, + camera_to_worlds: TensorType["num_cameras", 3, 4], + fx: Union[TensorType["num_cameras"], float], + fy: Union[TensorType["num_cameras"], float], + cx: float, + cy: float, + distortion_params: Optional[TensorType["num_cameras", 6]] = None, + camera_type: CameraType = CameraType.PERSPECTIVE, + ): + self._num_cameras = camera_to_worlds.shape[0] + self.camera_to_worlds = camera_to_worlds + if not isinstance(fx, torch.Tensor): + fx = torch.Tensor([fx]) + if not isinstance(fy, torch.Tensor): + fy = torch.Tensor([fy]) + self.fx = fx.broadcast_to((self._num_cameras)).to(self.device) + self.fy = fy.broadcast_to((self._num_cameras)).to(self.device) + self.cx = cx + self.cy = cy + if distortion_params is not None: + self.distortion_params = distortion_params.broadcast_to((self._num_cameras, 6)) + else: + self.distortion_params = None + self._image_heights = int(self.cy * 2) + self._image_widths = int(self.cx * 2) + self.camera_type = camera_type @property def device(self): """Returns the device that the camera is on.""" - return self.camera_to_world.device + return self.camera_to_worlds.device - @abstractmethod - def get_num_intrinsics_params(self) -> int: - """ - Returns: - number of optimizable intrinsic parameters - """ - return + @property + def size(self) -> int: + """Returns the number of cameras.""" + return self._num_cameras - @abstractmethod - def get_intrinsics(self) -> torch.Tensor: - """ - Returns: - Intrinsics matrix - """ - return + @property + def image_height(self) -> int: + """Returns the height of the images.""" + return self._image_heights - def get_camera_to_world(self) -> TensorType[3, 4]: - """ - Returns: - Camera to world transformation - """ - return self.camera_to_world + @property + def image_width(self) -> int: + """Returns the height of the images.""" + return self._image_widths - def get_camera_to_world_h(self) -> TensorType[4, 4]: + def to(self, device: Union[torch.device, str]) -> "Cameras": """ - Returns: - Camera to world transformation with homogeneous coordinates - """ - c2w = self.camera_to_world - ones = torch.tensor([0, 0, 0, 1], device=c2w.device)[None] - c2w = torch.cat([c2w, ones], dim=0) - return c2w - - @abstractmethod - def get_image_height(self) -> int: - """ - Returns: - Image height - """ - return - - @abstractmethod - def get_image_width(self) -> int: - """ - Returns: - Image width - """ - return - - @abstractmethod - def rescale_output_resolution(self, scaling_factor: float) -> None: - """Rescales the camera intrinsics for output resolution. - Args: - scaling_factor: Scaling factor + Device to move the camera to. Returns: - None + Cameras: Cameras on the specified device. """ - return + distortion_params = self.distortion_params.to(device) if self.distortion_params is not None else None + return Cameras( + camera_to_worlds=self.camera_to_worlds.to(device), + fx=self.fx.to(device), + fy=self.fy.to(device), + cx=self.cx, + cy=self.cy, + distortion_params=distortion_params, + camera_type=self.camera_type, + ) - def get_image_coords(self, pixel_offset: float = 0.5) -> TensorType["image_height", "image_width", 2]: + def get_image_coords(self, pixel_offset: float = 0.5) -> TensorType["height", "width", 2]: """ Args: pixel_offset (float): Offset for each pixel. Defaults to center of pixel (0.5) @@ -116,203 +125,105 @@ class Camera: Returns: TensorType["image_height", "image_width", 2]: Grid of image coordinates. """ - image_height = self.get_image_height() - image_width = self.get_image_width() + image_height = self.image_height + image_width = self.image_width image_coords = torch.meshgrid(torch.arange(image_height), torch.arange(image_width), indexing="ij") image_coords = torch.stack(image_coords, dim=-1) + pixel_offset # stored as (y, x) coordinates return image_coords - @classmethod - @abstractmethod def generate_rays( - cls, - intrinsics: TensorType[..., "num_intrinsics_params"], - camera_to_world: TensorType[..., 3, 4], - coords: TensorType[..., 2], + self, + camera_indices: Union[TensorType["num_rays":...], int], + coords: Optional[TensorType["num_rays":..., 2]] = None, + camera_to_world_delta: Optional[TensorType["num_rays":..., 3, 4]] = None, + distortion_params_delta: Optional[TensorType["num_rays":..., 6]] = None, ) -> RayBundle: - """ - Args: - intrinsics (TensorType[..., "num_intrinsics_params"]): Camera intrinsics - camera_to_world (TensorType[..., 3, 4]): Camera to world transformation matrix - coords (TensorType[..., 2]): Image grid coordinates + """Generates rays for the given camera indices. - Returns: - RayBundle: A bundle of rays for each grid coordinate. - """ - return - - def get_camera_ray_bundle(self, device=None) -> RayBundle: - """Generate rays for the camera. - - Returns: - Rays: Camera rays of shape [height, width] - """ - if device is None: - device = self.camera_to_world.device - height = self.get_image_height() - width = self.get_image_width() - intrinsics = ( - self.get_intrinsics().unsqueeze(0).repeat(height, width, 1).to(device) - ) # (num_rays, num_intrinsics_params) - camera_to_world = self.camera_to_world.unsqueeze(0).repeat(height, width, 1, 1).to(device) # (num_rays, 3, 4) - coords = self.get_image_coords().to(device) - ray_bundle = self.generate_rays(intrinsics, camera_to_world, coords) - if self.camera_index is not None: - ray_bundle.set_camera_indices(camera_index=self.camera_index) - return ray_bundle - - @abstractmethod - def to_json( - self, - image: Optional[TensorType["image_height", "image_width", 2]] = None, - resize_shape: Optional[Tuple[int, int]] = None, - ) -> Dict: - """Converts the camera to a json dictionary. + TODO - add support for intrinsics delta. + TODO - add support for distortions. Args: - image: An image in range [0, 1] that is encoded to a base64 string. Defaults to None. + camera_indices: Indices of the cameras to generate rays for. + coords: Coordinates of the pixels to generate rays for. If None, the full image will be rendered. + camera_to_world_delta: Optional delta for the camera to world matrices. + distortion_params_delta: Optional delta for the distortion parameters. Returns: - A JSON representation of the camera + Rays for the given camera indices and coords. """ - raise NotImplementedError - @staticmethod - def from_json(json_: Dict) -> "Camera": # pylint:disable=no-self-use - """Converts a json dictionary to the camera object. + if coords is None: + coords = self.get_image_coords().to(self.device) - Args: - json_: The json dictionary. + assert coords is not None + y = coords[..., 0] # (..., 1) + x = coords[..., 1] # (..., 1) + fx, fy = self.fx[camera_indices], self.fy[camera_indices] + cx, cy = self.cx, self.cy - Returns: - The camera object instantiated from the json dictionary. - """ - raise NotImplementedError - - -class PinholeCamera(Camera): - """Pinhole camera model. - - Args: - fx: Focal length in x direction - fy: Focal length in y direction - cx: Principal point in x direction - cy: Principal point in y direction - camera_to_world: Camera to world transformation - camera_index: Camera index. Defaults to None. - """ - - def __init__( - self, - cx: float, - cy: float, - fx: float, - fy: float, - camera_to_world: TensorType[3, 4] = torch.eye(4)[:3], - camera_index: Optional[int] = None, - ): - super().__init__(camera_to_world=camera_to_world, camera_index=camera_index) - self.cx = cx - self.cy = cy - self.fx = fx - self.fy = fy - - def get_num_intrinsics_params(self) -> int: - return 4 - - def get_image_width(self) -> int: - return int(self.cx * 2.0) - - def get_image_height(self) -> int: - return int(self.cy * 2.0) - - def get_intrinsics_matrix(self) -> TensorType[3, 3]: - """ - Returns: - TensorType[3, 3]: Pinhole camera intrinsics matrix - """ - K = torch.tensor( - [[self.fx, 0.0, self.cx], [0.0, self.fy, self.cy], [0, 0, 1.0]], - dtype=torch.float32, + directions = normalize(torch.stack([(x - cx) / fx, -(y - cy) / fy, -torch.ones_like(x)], -1), dim=-1) + directions_x_offset = normalize( + torch.stack([(x - cx + 1) / fx, -(y - cy) / fy, -torch.ones_like(x)], -1), dim=-1 + ) + directions_y_offset = normalize( + torch.stack([(x - cx) / fx, -(y - cy + 1) / fy, -torch.ones_like(x)], -1), dim=-1 ) - return K - def get_intrinsics(self) -> torch.Tensor: - return torch.tensor([self.cx, self.cy, self.fx, self.fy]) + directions_stack = torch.stack([directions, directions_x_offset, directions_y_offset], dim=0) - @classmethod - def fx_index(cls) -> int: - """Returns the index of the fx parameter in the intrinsics vector - TODO(ethan): redo this in a better way. - Ideally we can dynamically grab the focal length parameters depending on - Simple vs. not Simple Pinhole Model. - - Returns: - x focal length index - """ - return 2 - - @classmethod - def fy_index(cls) -> int: - """Returns the index of the fy parameter in the intrinsics vector - - Returns: - y focal length index - """ - return 3 - - def rescale_output_resolution(self, scaling_factor: float) -> None: - self.cx *= scaling_factor - self.cy *= scaling_factor - self.fx *= scaling_factor - self.fy *= scaling_factor - - @classmethod - def generate_rays( - cls, - intrinsics: TensorType[..., "num_intrinsics_params"], - camera_to_world: TensorType[..., 3, 4], - coords: TensorType[..., 2], - ) -> RayBundle: - - cx = intrinsics[..., 0:1] - cy = intrinsics[..., 1:2] - fx = intrinsics[..., cls.fx_index() : cls.fx_index() + 1] - fy = intrinsics[..., cls.fy_index() : cls.fy_index() + 1] - y = coords[..., 0:1] # (..., 1) - x = coords[..., 1:2] # (..., 1) - original_directions = torch.cat([(x - cx) / fx, -(y - cy) / fy, -torch.ones_like(x)], -1) # (..., 3) - rotation = camera_to_world[..., :3, :3] # (..., 3, 3) - directions = torch.sum( - original_directions[..., None, :] * rotation, dim=-1 + c2w = self.camera_to_worlds[camera_indices] + if camera_to_world_delta is not None: + c2w = c2w + camera_to_world_delta + rotation = c2w[..., :3, :3] # (..., 3, 3) + directions_stack = torch.sum( + directions_stack[..., None, :] * rotation, dim=-1 ) # (..., 1, 3) * (..., 3, 3) -> (..., 3) - directions = normalize(directions, dim=-1) - origins = camera_to_world[..., :3, 3] # (..., 3) + directions_stack = normalize(directions_stack, dim=-1) - ## Calculate pixel area directly - dirx_min = normalize(torch.cat([(x - cx - 0.5) / fx, -(y - cy) / fy, -torch.ones_like(x)], -1), dim=-1) - dirx_max = normalize(torch.cat([(x - cx + 0.5) / fx, -(y - cy) / fy, -torch.ones_like(x)], -1), dim=-1) - diry_min = normalize(torch.cat([(x - cx) / fx, -(y - cy - 0.5) / fy, -torch.ones_like(x)], -1), dim=-1) - diry_max = normalize(torch.cat([(x - cx) / fx, -(y - cy + 0.5) / fy, -torch.ones_like(x)], -1), dim=-1) - dx = torch.sqrt(torch.sum((dirx_max - dirx_min) ** 2, dim=-1)) - dy = torch.sqrt(torch.sum((diry_max - diry_min) ** 2, dim=-1)) + distortion_params = None + if self.distortion_params is not None: + if distortion_params_delta is not None: + distortion_params = self.distortion_params[camera_indices] + distortion_params_delta + elif distortion_params_delta is not None: + distortion_params = distortion_params_delta + + if distortion_params is not None: + raise NotImplementedError("Camera distortion not implemented.") + + origins = c2w[..., :3, 3] # (..., 3) + directions = directions_stack[0] + + dx = torch.sqrt(torch.sum((directions - directions_stack[1]) ** 2, dim=-1)) + dy = torch.sqrt(torch.sum((directions - directions_stack[2]) ** 2, dim=-1)) pixel_area = dx * dy return RayBundle(origins=origins, directions=directions, pixel_area=pixel_area[..., None]) def to_json( self, - image: Optional[TensorType["image_height", "image_width", 2]] = None, + camera_idx: int, + image: Optional[TensorType["height", "width", 2]] = None, resize_shape: Optional[Tuple[int, int]] = None, ) -> Dict: + """Convert a camera to a json dictionary. + + Args: + camera_idx (int): Index of the camera to convert. + image: An image in range [0, 1] that is encoded to a base64 string. Defaults to None. + resize_shape: Shape to resize the image to. Defaults to None. + + Returns: + A JSON representation of the camera + """ json_ = { "type": "PinholeCamera", "cx": self.cx, "cy": self.cy, - "fx": self.fx, - "fy": self.fy, - "camera_to_world": self.camera_to_world.tolist(), - "camera_index": self.camera_index, + "fx": self.fx[camera_idx].tolist(), + "fy": self.fy[camera_idx].tolist(), + "camera_to_world": self.camera_to_worlds[camera_idx].tolist(), + "camera_index": camera_idx, } if image is not None: image_uint8 = (image * 255).detach().cpu().numpy().astype(np.uint8) @@ -322,187 +233,27 @@ class PinholeCamera(Camera): json_["image"] = str("data:image/png;base64," + base64.b64encode(data).decode("ascii")) return json_ - @staticmethod - def from_json(json_: Dict) -> "PinholeCamera": # pylint:disable=no-self-use - raise NotImplementedError + def get_intrinsics_matrices(self) -> TensorType["num_cameras", 3, 3]: + """Returns the intrinsic matrices for each camera. - -class SimplePinholeCamera(PinholeCamera): - """Simple Pinhole Camera model. - - Args: - cx: Principal point x direction - cy: Principal point y direction - f: Focal length - camera_to_world: Camera to world transformation matrix - camera_index: Camera index. Defaults to None. - """ - - def __init__( - self, - cx: float, - cy: float, - f: float, - camera_to_world: TensorType[3, 4] = torch.eye(4)[:3], - camera_index: Optional[int] = None, - ): - super().__init__(cx, cy, f, f, camera_to_world=camera_to_world, camera_index=camera_index) - - @classmethod - def fx_index(cls): - return 2 - - @classmethod - def fy_index(cls): - return 2 - - @staticmethod - def from_json(json_: Dict) -> "SimplePinholeCamera": # pylint:disable=no-self-use - raise NotImplementedError - - -class EquirectangularCamera(Camera): - """Equirectangular (360 degree) camera model. - - Args: - height: Height of the image - width: Width of the image - camera_to_world: Camera to world transformation matrix - camera_index: Camera index. Defaults to None. - """ - - def __init__( - self, - height: int, - width: int, - camera_to_world: TensorType[3, 4] = torch.eye(4)[:3], - camera_index: Optional[int] = None, - ): - super().__init__(camera_to_world=camera_to_world, camera_index=camera_index) - self.height = height - self.width = width - - def get_num_intrinsics_params(self): - return 2 - - def get_image_width(self): - return self.width - - def get_image_height(self): - return self.height - - def get_intrinsics(self) -> torch.Tensor: - return torch.tensor([self.height, self.width]) + Returns: + Pinhole camera intrinsics matrices + """ + K = torch.zeros((self.size, 3, 3), dtype=torch.float32) + K[:, 0, 0] = self.fx + K[:, 1, 1] = self.fy + K[:, 0, 2] = self.cx + K[:, 1, 2] = self.cy + K[:, 2, 2] = 1.0 + return K def rescale_output_resolution(self, scaling_factor: float) -> None: - self.height *= scaling_factor - self.width *= scaling_factor + """Rescale the output resolution of the cameras. - @classmethod - def generate_rays( - cls, - intrinsics: TensorType[..., 2], - camera_to_world: TensorType[..., 3, 4], - coords: TensorType[..., 2], - ) -> RayBundle: - - y = coords[..., 0:1] # (num_rays, 1) - x = coords[..., 1:2] # (num_rays, 1) - - height = intrinsics[..., 0:1] - width = intrinsics[..., 1:2] - - phi = y / height * torch.pi - theta = -x / width * 2 * torch.pi - - d_phi_min = (y + 0.5) / height * torch.pi - d_phi_max = (y - 0.5) / height * torch.pi - d_theta_min = -(x - 0.5) / width * 2 * torch.pi - d_theta_max = -(x + 0.5) / width * 2 * torch.pi - - s_phi = torch.sin(phi) - c_phi = torch.cos(phi) - s_theta = torch.sin(theta) - c_theta = torch.cos(theta) - - directions = torch.cat([c_theta * s_phi, c_phi, s_theta * s_phi], dim=-1) - origins = camera_to_world[..., :3, 3] # (num_rays, 3) - - ## Calculate area directly - dirx_min = normalize( - torch.cat([torch.cos(d_theta_min) * s_phi, c_phi, torch.sin(d_theta_min) * s_phi], -1), dim=-1 - ) - dirx_max = normalize( - torch.cat([torch.cos(d_theta_max) * s_phi, c_phi, torch.sin(d_theta_max) * s_phi], -1), dim=-1 - ) - diry_min = normalize( - torch.cat([c_theta * torch.sin(d_phi_min), torch.cos(d_phi_min), s_theta * torch.sin(d_phi_min)], -1), - dim=-1, - ) - diry_max = normalize( - torch.cat([c_theta * torch.sin(d_phi_max), torch.cos(d_phi_max), s_theta * torch.sin(d_phi_max)], -1), - dim=-1, - ) - dx = torch.sqrt(torch.sum((dirx_max - dirx_min) ** 2, dim=-1)) - dy = torch.sqrt(torch.sum((diry_max - diry_min) ** 2, dim=-1)) - pixel_area = dx * dy - return RayBundle(origins=origins, directions=directions, pixel_area=pixel_area[..., None]) - - def to_json( - self, - image: Optional[TensorType["image_height", "image_width", 2]] = None, - resize_shape: Optional[Tuple[int, int]] = None, - ) -> Dict: - raise NotImplementedError - - @staticmethod - def from_json(json_: Dict) -> "SimplePinholeCamera": # pylint:disable=no-self-use - raise NotImplementedError - - -def get_intrinsics_from_intrinsics_matrix(intrinsics_matrix: TensorType[3, 3]): - """ - Return a flat intrinsics tensor. - """ - fx = intrinsics_matrix[0, 0] - fy = intrinsics_matrix[1, 1] - cx = intrinsics_matrix[0, 2] - cy = intrinsics_matrix[1, 2] - if fx == fy: - return torch.tensor([cx, cy, fx]) - return torch.tensor([cx, cy, fx, fy]) - - -def get_camera_model(num_intrinsics_params: int) -> Type[Camera]: - # TODO: we should specify the type in the dataloading pipeline instead of relying on number of intrinsics - # to make the choice - """Returns the camera model given the specified number of intrinsics parameters. - - Args: - num_intrinsics_params (int): Number of intrinsic parametes. - - Returns: - Camera: Camera model - """ - if num_intrinsics_params == 2: - return EquirectangularCamera - if num_intrinsics_params == 3: - return SimplePinholeCamera - if num_intrinsics_params == 4: - return PinholeCamera - raise NotImplementedError - - -def get_camera(intrinsics: TensorType["num_intrinsics"], camera_to_world: TensorType[3, 4], camera_index=None): - """Returns a transformed camera. - - Args: - intrinsics (TensorType["num_intrinsics"]): Intrinsics tensor. - camera_to_world (TensorType[3, 4]): Camera to world transformation. - camera_index (int): Camera index. - """ - assert len(intrinsics.shape) == 1, "The intrinsics object should be a flat tensor." - num_intrinsics_params = len(intrinsics) - camera_class = get_camera_model(num_intrinsics_params) - camera = camera_class(*intrinsics.tolist(), camera_to_world=camera_to_world, camera_index=camera_index) - return camera + Args: + scaling_factor: Scaling factor to apply to the output resolution. + """ + self.fx *= scaling_factor + self.fy *= scaling_factor + self.cx *= scaling_factor + self.cy *= scaling_factor diff --git a/nerfactory/cameras/utils.py b/nerfactory/cameras/utils.py index 5858fb6b8..56f726f2f 100644 --- a/nerfactory/cameras/utils.py +++ b/nerfactory/cameras/utils.py @@ -18,9 +18,11 @@ Camera transformation helper code. """ import math +from typing import Tuple import numpy as np import torch +from torchtyping import TensorType _EPS = np.finfo(float).eps * 4.0 @@ -257,24 +259,30 @@ def get_interpolated_K(KA, KB, steps=10): return Ks -def get_interpolated_poses_many(list_of_poses, list_of_Ks, steps_per_transition=10): - """ +def get_interpolated_poses_many( + poses: TensorType["num_poses", 3, 4], + Ks: TensorType["num_poses", 3, 3], + steps_per_transition=10, +) -> Tuple[TensorType["num_poses", 3, 4], TensorType["num_poses", 3, 3]]: + """Return interpolated poses for many camera poses. + Args: - list_of_poses - steps - distances_per_step + poses: list of camera poses + Ks: list of camera intrinsics + steps_per_transition: number of steps per transition + Returns: + tuple of new poses and intrinsics """ traj = [] Ks = [] - for idx in range(len(list_of_poses) - 1): - poseA = list_of_poses[idx] - poseB = list_of_poses[idx + 1] - # steps = int(np.linalg.norm(poseA[:3,3] - poseB[:3,3], ord=2) / distance_per_step) + for idx in range(poses.shape[0] - 1): + poseA = poses[idx] + poseB = poses[idx + 1] posesAB = get_interpolated_poses(poseA, poseB, steps=steps_per_transition) traj += posesAB - Ks += get_interpolated_K(list_of_Ks[idx], list_of_Ks[idx + 1], steps_per_transition) - return traj, Ks + Ks += get_interpolated_K(Ks[idx], Ks[idx + 1], steps_per_transition) + return torch.stack(traj, dim=0), torch.stack(Ks, dim=0) def get_swirl_poses(pose): diff --git a/nerfactory/dataloaders/base.py b/nerfactory/dataloaders/base.py index 8302cbd70..3825519a7 100644 --- a/nerfactory/dataloaders/base.py +++ b/nerfactory/dataloaders/base.py @@ -210,12 +210,14 @@ class VanillaDataloader(Dataloader): # pylint: disable=abstract-method dataset_train = config.train_dataset.setup() self.train_datasetinputs = dataset_train.get_dataset_inputs(split="train") + self.train_cameras = self.train_datasetinputs.cameras.to(device) if config.eval_dataset is not None: dataset_eval = config.eval_dataset.setup() else: logging.info("No eval dataset specified so using train dataset for eval.") dataset_eval = dataset_train self.config.eval_datasetinputs = dataset_eval.get_dataset_inputs(split="val" if not test_mode else "test") + self.eval_cameras = self.config.eval_datasetinputs.cameras.to(device) use_train = self.train_datasetinputs is not None use_eval = self.config.eval_datasetinputs is not None super().__init__(use_train, use_eval) @@ -233,9 +235,7 @@ class VanillaDataloader(Dataloader): # pylint: disable=abstract-method ) # TODO(ethan): pass this in self.iter_train_image_sampler = iter(self.train_image_sampler) self.train_pixel_sampler = PixelSampler(self.config.train_num_rays_per_batch) - self.train_ray_generator = RayGenerator( - self.train_datasetinputs.intrinsics, self.train_datasetinputs.camera_to_world - ) + self.train_ray_generator = RayGenerator(self.train_cameras) def setup_eval(self): """Sets up the dataloader for evaluation""" @@ -245,8 +245,7 @@ class VanillaDataloader(Dataloader): # pylint: disable=abstract-method self.eval_image_dataset = PanopticImageDataset(**self.config.eval_datasetinputs.as_dict()) self.eval_dataloader = FixedIndicesEvalDataloader( image_dataset=self.eval_image_dataset, - intrinsics=self.config.eval_datasetinputs.intrinsics, - camera_to_world=self.config.eval_datasetinputs.camera_to_world, + cameras=self.eval_cameras, num_rays_per_chunk=self.config.eval_num_rays_per_chunk, image_indices=self.config.eval_image_indices, device=self.device, @@ -259,7 +258,7 @@ class VanillaDataloader(Dataloader): # pylint: disable=abstract-method image_batch = next(self.iter_train_image_sampler) batch = self.train_pixel_sampler.sample(image_batch) ray_indices = batch["indices"] - ray_bundle = self.train_ray_generator.forward(ray_indices) + ray_bundle = self.train_ray_generator(ray_indices) return ray_bundle, batch def next_eval(self) -> Tuple[RayBundle, Dict]: diff --git a/nerfactory/dataloaders/datasets.py b/nerfactory/dataloaders/datasets.py index 283635723..a0f60733b 100644 --- a/nerfactory/dataloaders/datasets.py +++ b/nerfactory/dataloaders/datasets.py @@ -27,6 +27,7 @@ import numpy as np import torch from scipy.spatial.transform import Rotation +from nerfactory.cameras.cameras import Cameras, CameraType from nerfactory.configs import base as cfg from nerfactory.dataloaders.colmap_utils import ( read_cameras_binary, @@ -167,21 +168,25 @@ class Blender(Dataset): cx = image_width / 2.0 cy = image_height / 2.0 camera_to_world = torch.from_numpy(poses[:, :3]) # camera to world transform - num_cameras = len(image_filenames) - num_intrinsics_params = 3 - intrinsics = torch.ones((num_cameras, num_intrinsics_params), dtype=torch.float32) - intrinsics *= torch.tensor([cx, cy, focal_length]) # in x,y,z order camera_to_world[..., 3] *= self.scale_factor scene_bounds = SceneBounds(aabb=torch.tensor([[-1.5, -1.5, -1.5], [1.5, 1.5, 1.5]], dtype=torch.float32)) + cameras = Cameras( + camera_to_worlds=camera_to_world, + fx=focal_length / self.downscale_factor, + fy=focal_length / self.downscale_factor, + cx=cx / self.downscale_factor, + cy=cy / self.downscale_factor, + camera_type=CameraType.PERSPECTIVE, + ) + dataset_inputs = DatasetInputs( image_filenames=image_filenames, + cameras=cameras, downscale_factor=self.downscale_factor, alpha_color=alpha_color_tensor, - intrinsics=intrinsics * 1.0 / self.downscale_factor, # downscaling the intrinsics here - camera_to_world=camera_to_world, scene_bounds=scene_bounds, ) @@ -252,12 +257,20 @@ class InstantNGP(Dataset): ) ) + cameras = Cameras( + fx=focal_length / self.downscale_factor, + fy=focal_length / self.downscale_factor, + cx=cx / self.downscale_factor, + cy=cy / self.downscale_factor, + camera_to_worlds=camera_to_world, + camera_type=CameraType.PERSPECTIVE, + ) + # TODO(ethan): add alpha background color dataset_inputs = DatasetInputs( image_filenames=image_filenames, downscale_factor=self.downscale_factor, - intrinsics=intrinsics * 1.0 / self.downscale_factor, # downscaling the intrinsics here - camera_to_world=camera_to_world, + cameras=cameras, scene_bounds=scene_bounds, ) @@ -371,11 +384,19 @@ class Mipnerf360(Dataset): aabb = torch.tensor([[-4, -4, -4], [4, 4, 4]], dtype=torch.float32) * self.aabb_scale scene_bounds = SceneBounds(aabb=aabb) + cameras = Cameras( + fx=focal_length / self.downscale_factor, + fy=focal_length / self.downscale_factor, + cx=cx / self.downscale_factor, + cy=cy / self.downscale_factor, + camera_to_worlds=camera_to_world, + camera_type=CameraType.PERSPECTIVE, + ) + dataset_inputs = DatasetInputs( image_filenames=image_filenames, downscale_factor=1, - intrinsics=intrinsics, - camera_to_world=camera_to_world, + cameras=cameras, scene_bounds=scene_bounds, ) @@ -487,11 +508,19 @@ class Record3D(Dataset): aabb = torch.tensor([[-1, -1, -1], [1, 1, 1]], dtype=torch.float32) * self.aabb_scale scene_bounds = SceneBounds(aabb=aabb) + cameras = Cameras( + fx=focal_length / self.downscale_factor, + fy=focal_length / self.downscale_factor, + cx=cx / self.downscale_factor, + cy=cy / self.downscale_factor, + camera_to_worlds=camera_to_world, + camera_type=CameraType.PERSPECTIVE, + ) + dataset_inputs = DatasetInputs( image_filenames=image_filenames, downscale_factor=self.downscale_factor, - intrinsics=intrinsics * 1.0 / self.downscale_factor, - camera_to_world=camera_to_world, + cameras=cameras, scene_bounds=scene_bounds, ) @@ -562,15 +591,16 @@ class Friends(Dataset): # --- intrinsics --- cameras_data = read_cameras_binary(os.path.join(abs_dir, "colmap", "cameras.bin")) - intrinsics = [] + focal_lengths = [] for image_path in image_paths: cam = cameras_data[image_path_to_image_id[image_path]] assert len(cam.params) == 3 - focal_length = cam.params[0] # f (fx and fy) - cx = cam.params[1] # cx - cy = cam.params[2] # cy - intrinsics.append([cx, cy, focal_length]) - intrinsics = torch.tensor(intrinsics).float() + focal_lengths.append(cam.params[0]) # f (fx and fy) + focal_lengths = torch.stack(focal_lengths, dim=0) + + cam = cameras_data[image_path_to_image_id[image_paths[0]]] + cx = cam.params[1] # cx + cy = cam.params[2] # cy # --- camera_to_world (extrinsics) --- camera_to_world = [] @@ -626,11 +656,19 @@ class Friends(Dataset): point_cloud.xyz = xyz point_cloud.rgb = rgb + cameras = Cameras( + fx=focal_lengths / self.downscale_factor, + fy=focal_lengths / self.downscale_factor, + cx=cx / self.downscale_factor, + cy=cy / self.downscale_factor, + camera_to_worlds=camera_to_world, + camera_type=CameraType.PERSPECTIVE, + ) + dataset_inputs = DatasetInputs( image_filenames=image_filenames, downscale_factor=self.downscale_factor, - intrinsics=intrinsics / self.downscale_factor, - camera_to_world=camera_to_world, + cameras=cameras, semantics=semantics, point_cloud=point_cloud, scene_bounds=scene_bounds, diff --git a/nerfactory/dataloaders/eval.py b/nerfactory/dataloaders/eval.py index 5e589686c..9355f4c80 100644 --- a/nerfactory/dataloaders/eval.py +++ b/nerfactory/dataloaders/eval.py @@ -22,7 +22,7 @@ from typing import Dict, Optional, Tuple, Union import torch -from nerfactory.cameras.cameras import Camera, get_camera +from nerfactory.cameras.cameras import Cameras from nerfactory.cameras.rays import RayBundle from nerfactory.dataloaders.image_dataset import ImageDataset from nerfactory.utils.misc import get_dict_to_torch @@ -34,16 +34,14 @@ class EvalDataloader: # pylint: disable=too-few-public-methods def __init__( self, image_dataset: ImageDataset, - intrinsics, - camera_to_world, + cameras, num_rays_per_chunk: int, device: Union[torch.device, str] = "cpu", **kwargs, ): super().__init__() self.image_dataset = image_dataset - self.intrinsics = intrinsics - self.camera_to_world = camera_to_world + self.cameras = cameras self.num_rays_per_chunk = num_rays_per_chunk self.device = device self.kwargs = kwargs @@ -57,18 +55,24 @@ class EvalDataloader: # pylint: disable=too-few-public-methods def __next__(self) -> Tuple[RayBundle, Dict]: """Returns the next batch of data""" - def get_camera(self, image_idx) -> Camera: + def get_camera(self, image_idx) -> Cameras: """Get camera for the given image index""" - intrinsics = self.intrinsics[image_idx].to(self.device) - camera_to_world = self.camera_to_world[image_idx].to(self.device) - camera = get_camera(intrinsics, camera_to_world, camera_index=image_idx) + camera = Cameras( + fx=self.cameras.fx[image_idx], + fy=self.cameras.fy[image_idx], + cx=self.cameras.cx, + cy=self.cameras.cy, + camera_to_worlds=self.cameras.camera_to_worlds[image_idx], + distortion_params=self.cameras.distortion_params[image_idx], + camera_type=self.cameras.camera_type, + ) return camera def get_data_from_image_idx(self, image_idx) -> Tuple[RayBundle, Dict]: """Returns the data for a specific image index.""" - camera = self.get_camera(image_idx) - ray_bundle = camera.get_camera_ray_bundle(device=self.device) + ray_bundle = self.cameras.generate_rays(camera_indices=image_idx) ray_bundle.num_rays_per_chunk = self.num_rays_per_chunk + ray_bundle.camera_indices = torch.Tensor([image_idx])[..., None].int() batch = self.image_dataset[image_idx] batch = get_dict_to_torch(batch, device=self.device) return ray_bundle, batch @@ -80,8 +84,7 @@ class FixedIndicesEvalDataloader(EvalDataloader): def __init__( self, image_dataset: ImageDataset, - intrinsics, - camera_to_world, + cameras: Cameras, num_rays_per_chunk: int, image_indices: Optional[Tuple[int]] = None, device: Union[torch.device, str] = "cpu", @@ -92,7 +95,7 @@ class FixedIndicesEvalDataloader(EvalDataloader): image_dataset: ImageDataset to load data from image_indices: List of image indices to load data from. If None, then use all images. """ - super().__init__(image_dataset, intrinsics, camera_to_world, num_rays_per_chunk, device, **kwargs) + super().__init__(image_dataset, cameras, num_rays_per_chunk, device, **kwargs) if image_indices is None: self.image_indices = list(range(len(image_dataset))) else: @@ -120,13 +123,12 @@ class RandIndicesEvalDataloader(EvalDataloader): def __init__( self, image_dataset: ImageDataset, - intrinsics, - camera_to_world, + cameras: Cameras, num_rays_per_chunk: int, device: Union[torch.device, str] = "cpu", **kwargs, ): - super().__init__(image_dataset, intrinsics, camera_to_world, num_rays_per_chunk, device, **kwargs) + super().__init__(image_dataset, cameras, num_rays_per_chunk, device, **kwargs) self.count = 0 def __iter__(self): @@ -135,7 +137,7 @@ class RandIndicesEvalDataloader(EvalDataloader): def __next__(self): if self.count < 1: - image_indices = range(len(self.camera_to_world)) + image_indices = range(self.cameras.size) image_idx = random.choice(image_indices) ray_bundle, batch = self.get_data_from_image_idx(image_idx) self.count += 1 diff --git a/nerfactory/dataloaders/structs.py b/nerfactory/dataloaders/structs.py index af17a605a..5f85b5d77 100644 --- a/nerfactory/dataloaders/structs.py +++ b/nerfactory/dataloaders/structs.py @@ -22,6 +22,7 @@ from typing import Dict, List, Optional, Union import torch from torchtyping import TensorType +from nerfactory.cameras.cameras import Cameras from nerfactory.cameras.rays import RayBundle @@ -119,8 +120,7 @@ class DatasetInputs: """ image_filenames: List[str] - intrinsics: TensorType["num_cameras", "num_intrinsics_params"] - camera_to_world: TensorType["num_cameras", 3, 4] + cameras: Cameras downscale_factor: int = 1 mask_filenames: Optional[List[str]] = None depth_filenames: Optional[List[str]] = None @@ -129,19 +129,6 @@ class DatasetInputs: point_cloud: PointCloud = PointCloud() alpha_color: Optional[TensorType[3]] = None - def check_inputs(self): - """Check the inputs to make sure everything is okay.""" - assert self.intrinsics.dtype == torch.float32 - assert self.camera_to_world.dtype == torch.float32 - - def save_to_folder_name(self, data_directory: str): - """Save the dataset inputs.""" - raise NotImplementedError - - def load_from_folder_name(self, data_directory: str): - """Load the saved dataset inputs.""" - raise NotImplementedError - def as_dict(self) -> dict: """Returns the dataclass as a dictionary.""" return vars(self) diff --git a/nerfactory/models/base.py b/nerfactory/models/base.py index 9ba18ea17..bf34a7fec 100644 --- a/nerfactory/models/base.py +++ b/nerfactory/models/base.py @@ -26,7 +26,6 @@ import torch from torch import nn from torch.nn import Parameter -from nerfactory.cameras.cameras import Camera from nerfactory.cameras.rays import RayBundle from nerfactory.configs import base as cfg from nerfactory.dataloaders.structs import SceneBounds @@ -198,11 +197,6 @@ class Model(nn.Module): outputs[output_name] = torch.cat(outputs_list).view(image_height, image_width, -1) # type: ignore return outputs - def get_outputs_for_camera(self, camera: Camera): - """Get the model outputs for a Camera.""" - camera_ray_bundle = camera.get_camera_ray_bundle(device=self.device) - return self.get_outputs_for_camera_ray_bundle(camera_ray_bundle) - @abstractmethod def log_test_image_outputs(self, image_idx, step, batch, outputs) -> float: """Writes the test image outputs. diff --git a/nerfactory/models/modules/ray_generator.py b/nerfactory/models/modules/ray_generator.py index a6393da27..8b521648a 100644 --- a/nerfactory/models/modules/ray_generator.py +++ b/nerfactory/models/modules/ray_generator.py @@ -18,7 +18,7 @@ Ray generator. from torch import nn from torchtyping import TensorType -from nerfactory.cameras.cameras import get_camera_model +from nerfactory.cameras.cameras import Cameras from nerfactory.cameras.rays import RayBundle @@ -31,23 +31,10 @@ class RayGenerator(nn.Module): camera_to_world: Camera to world transformation matrix. """ - def __init__( - self, - intrinsics: TensorType["num_cameras", "num_intrinsics_params"], - camera_to_world: TensorType["num_cameras", 3, 4], - ) -> None: + def __init__(self, cameras: Cameras) -> None: super().__init__() - self.num_cameras, self.num_intrinsics_params = intrinsics.shape - assert self.num_cameras >= 0 - self.intrinsics = nn.Parameter(intrinsics, requires_grad=False) - self.camera_to_world = nn.Parameter(camera_to_world, requires_grad=False) - # TODO(ethan): add learnable parameters that are deltas on the intrinsics and camera_to_world parameters - - # NOTE(ethan): we currently assume all images have the same height and width - camera_index = 0 - self.camera_class = get_camera_model(self.num_intrinsics_params) - camera = self.camera_class(*self.intrinsics[camera_index].tolist()) - self.image_coords = nn.Parameter(camera.get_image_coords(), requires_grad=False) + self.cameras = cameras + self.image_coords = nn.Parameter(cameras.get_image_coords(), requires_grad=False) def forward(self, ray_indices: TensorType["num_rays", 3]) -> RayBundle: """Index into the cameras to generate the rays. @@ -58,12 +45,11 @@ class RayGenerator(nn.Module): c = ray_indices[:, 0] # camera indices y = ray_indices[:, 1] # row indices x = ray_indices[:, 2] # col indices - intrinsics = self.intrinsics[c] - camera_to_world = self.camera_to_world[c] coords = self.image_coords[y, x] - ray_bundle = self.camera_class.generate_rays( - intrinsics=intrinsics, camera_to_world=camera_to_world, coords=coords + ray_bundle = self.cameras.generate_rays( + camera_indices=c, + coords=coords, ) ray_bundle.camera_indices = c[..., None] # ["num_rays",1] return ray_bundle diff --git a/nerfactory/utils/plotly.py b/nerfactory/utils/plotly.py index 811c5c8e7..fbb631d34 100644 --- a/nerfactory/utils/plotly.py +++ b/nerfactory/utils/plotly.py @@ -30,7 +30,7 @@ import torch.nn.functional as F from plotly import express as ex from torchtyping import TensorType -from nerfactory.cameras.cameras import Camera +from nerfactory.cameras.cameras import Cameras from nerfactory.cameras.rays import Frustums, RayBundle from nerfactory.utils.math import Gaussians @@ -444,7 +444,7 @@ def get_ray_bundle_lines( ) -def vis_camera_rays(camera: Camera) -> go.Figure: # type: ignore +def vis_camera_rays(cameras: Cameras) -> go.Figure: # type: ignore """Visualize camera rays. Args: @@ -454,12 +454,12 @@ def vis_camera_rays(camera: Camera) -> go.Figure: # type: ignore Plotly lines """ - coords = camera.get_image_coords() - coords[..., 0] /= camera.get_image_height() - coords[..., 1] /= camera.get_image_width() + coords = cameras.get_image_coords() + coords[..., 0] /= cameras.image_height + coords[..., 1] /= cameras.image_width coords = torch.cat([coords, torch.ones((*coords.shape[:-1], 1))], dim=-1) - ray_bundle = camera.get_camera_ray_bundle() + ray_bundle = cameras.generate_rays(camera_indices=0) origins = ray_bundle.origins.view(-1, 3) directions = ray_bundle.directions.view(-1, 3) @@ -498,7 +498,7 @@ def vis_camera_rays(camera: Camera) -> go.Figure: # type: ignore return fig -def get_camera_frustums(cameras: List[Camera]): +def get_camera_frustums(cameras: Cameras): """Returns the camera frustums for the cameras that we are using. Args: @@ -507,7 +507,7 @@ def get_camera_frustums(cameras: List[Camera]): Returns: A plotly scatter that can be plotted. """ - for camera in cameras: - json_ = camera.to_json() + for camera_idx in range(cameras.size): + json_ = cameras.to_json(camera_idx=camera_idx) print(json_) raise NotImplementedError diff --git a/nerfactory/viewer/server/viewer_utils.py b/nerfactory/viewer/server/viewer_utils.py index 1485ea2cc..b65a98674 100644 --- a/nerfactory/viewer/server/viewer_utils.py +++ b/nerfactory/viewer/server/viewer_utils.py @@ -25,7 +25,7 @@ from typing import Any, Dict import numpy as np import torch -from nerfactory.cameras.cameras import get_camera, get_intrinsics_from_intrinsics_matrix +from nerfactory.cameras.cameras import Cameras from nerfactory.cameras.rays import RayBundle from nerfactory.configs import base as cfg from nerfactory.dataloaders.image_dataset import ImageDataset @@ -225,9 +225,9 @@ class VisualizerState: image_indices = range(len(image_dataset)) for idx in image_indices: image = image_dataset[idx]["image"] - camera = get_camera(dataset_inputs.intrinsics[idx], dataset_inputs.camera_to_world[idx], None) bgr = image[..., [2, 1, 0]] - self.vis[f"sceneState/cameras/{idx:06d}"].write(camera.to_json(image=bgr, resize_shape=(100, 100))) + camera_json = dataset_inputs.cameras.to_json(camera_idx=idx, image=bgr, resize_shape=(100, 100)) + self.vis[f"sceneState/cameras/{idx:06d}"].write(camera_json) # draw the scene bounds (i.e., the bounding box) json_ = dataset_inputs.scene_bounds.to_json() @@ -451,9 +451,17 @@ class VisualizerState: ], dim=0, ) - intrinsics = get_intrinsics_from_intrinsics_matrix(intrinsics_matrix) - camera = get_camera(intrinsics, camera_to_world) - camera_ray_bundle = camera.get_camera_ray_bundle(device=graph.device) + + camera = Cameras( + fx=intrinsics_matrix[0, 0], + fy=intrinsics_matrix[1, 1], + cx=intrinsics_matrix[0, 2], + cy=intrinsics_matrix[1, 2], + camera_to_worlds=camera_to_world[None, ...], + ) + camera = camera.to(graph.device) + + camera_ray_bundle = camera.generate_rays(camera_indices=0) camera_ray_bundle.num_rays_per_chunk = self.config.num_rays_per_chunk graph.eval() diff --git a/scripts/run_eval.py b/scripts/run_eval.py index 9935af834..205d4dc17 100644 --- a/scripts/run_eval.py +++ b/scripts/run_eval.py @@ -13,10 +13,10 @@ import torch from tqdm import tqdm from nerfactory.cameras.camera_paths import ( - CameraPath, get_interpolated_camera_path, get_spiral_path, ) +from nerfactory.cameras.cameras import Cameras from nerfactory.configs import base as cfg from nerfactory.configs.utils import cli_from_base_configs from nerfactory.pipelines.base import Pipeline @@ -88,7 +88,7 @@ def render_stats_dict(pipeline: Pipeline) -> Dict[str, float]: def render_trajectory_video( pipeline: Pipeline, - camera_path: CameraPath, + cameras: Cameras, output_filename: Path, rendered_output_name: str, rendered_resolution_scaling_factor: float = 1.0, @@ -98,7 +98,7 @@ def render_trajectory_video( Args: pipeline: Pipeline to evaluate with. - camera_path: Index of the image to render. + cameras: Cameras to render. output_filename: Name of the output file. rendered_output_name: Name of the renderer output to use. rendered_resolution_scaling_factor: Scaling factor to apply to the camera image resolution. @@ -107,9 +107,9 @@ def render_trajectory_video( """ print("Creating trajectory video.") images = [] - for camera in tqdm(camera_path.cameras): - camera.rescale_output_resolution(rendered_resolution_scaling_factor) - camera_ray_bundle = camera.get_camera_ray_bundle().to(pipeline.device) + cameras.rescale_output_resolution(rendered_resolution_scaling_factor) + for camera_idx in tqdm(range(cameras.size)): + camera_ray_bundle = cameras.generate_rays(camera_indices=camera_idx).to(pipeline.device) camera_ray_bundle.num_rays_per_chunk = num_rays_per_chunk with torch.no_grad(): outputs = pipeline.model.get_outputs_for_camera_ray_bundle(camera_ray_bundle) @@ -179,9 +179,8 @@ def main(config: cfg.Config): # TODO(ethan): pass in the up direction of the camera camera_path = get_spiral_path(camera_start, steps=30, radius=0.1) elif config.eval.traj == cfg.TrajectoryType.INTERP: - camera_start = pipeline.dataloader.eval_dataloader.get_camera(image_idx=0) - camera_end = pipeline.dataloader.eval_dataloader.get_camera(image_idx=10) - camera_path = get_interpolated_camera_path(camera_start, camera_end, steps=30) + cameras = pipeline.dataloader.eval_dataloader.get_camera(image_idx=[0, 10]) + camera_path = get_interpolated_camera_path(cameras, steps=30) else: raise NotImplementedError diff --git a/tests/cameras/test_cameras.py b/tests/cameras/test_cameras.py index fd01e2596..f3ca2eda5 100644 --- a/tests/cameras/test_cameras.py +++ b/tests/cameras/test_cameras.py @@ -4,40 +4,23 @@ Test the camera classes. import torch -from nerfactory.cameras.cameras import EquirectangularCamera, PinholeCamera +from nerfactory.cameras.cameras import Cameras from nerfactory.cameras.rays import RayBundle def test_pinhole_camera(): """Test that the pinhole camera model works.""" - pinhole_camera = PinholeCamera(cx=400.0, cy=400.0, fx=10.0, fy=10.0) - camera_ray_bundle = pinhole_camera.get_camera_ray_bundle() + c2w = torch.eye(4)[None, :3, :] + pinhole_camera = Cameras(cx=400.0, cy=400.0, fx=10.0, fy=10.0, camera_to_worlds=c2w) + camera_ray_bundle = pinhole_camera.generate_rays(camera_indices=0) assert isinstance(camera_ray_bundle, RayBundle) assert torch.allclose(camera_ray_bundle.origins[0], torch.tensor([0.0, 0.0, 0.0])) # Test generate rays on 1D input num_rays = 10 - intrinsics = torch.ones(num_rays, 4) # (num_rays, num_intrinsics_params) - camera_to_world = torch.ones(num_rays, 3, 4) # (num_rays, 3, 4) coords = torch.ones(num_rays, 2) - pinhole_camera.generate_rays(intrinsics, camera_to_world, coords) - - -def test_equirectangular_camera(): - """Test that the equirectangular camera model works.""" - equirectangular_camera = EquirectangularCamera(height=400, width=400) - camera_ray_bundle = equirectangular_camera.get_camera_ray_bundle() - assert isinstance(camera_ray_bundle, RayBundle) - assert torch.allclose(camera_ray_bundle.origins[0], torch.tensor([0.0, 0.0, 0.0])) - - # Test generate rays on 1D input - num_rays = 10 - intrinsics = torch.ones(num_rays, 2) * 20 # (num_rays, num_intrinsics_params) - camera_to_world = torch.ones(num_rays, 3, 4) # (num_rays, 3, 4) - coords = torch.ones(num_rays, 2) - equirectangular_camera.generate_rays(intrinsics, camera_to_world, coords) + pinhole_camera.generate_rays(camera_indices=0, coords=coords) if __name__ == "__main__": test_pinhole_camera() - test_equirectangular_camera()