From 8b9574bb5a33a21bd1efae9bd777328ee566c0a7 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Tue, 16 May 2023 18:47:46 -0700 Subject: [PATCH] Type fixes and cleanup (#1937) --- nerfstudio/cameras/camera_paths.py | 4 +- nerfstudio/cameras/camera_utils.py | 64 ++++++++++--------- nerfstudio/cameras/cameras.py | 21 +++--- .../data/dataparsers/nerfstudio_dataparser.py | 17 ++--- .../fields/nerfplayer_nerfacto_field.py | 3 +- nerfstudio/scripts/exporter.py | 6 +- nerfstudio/scripts/render.py | 6 +- 7 files changed, 66 insertions(+), 55 deletions(-) diff --git a/nerfstudio/cameras/camera_paths.py b/nerfstudio/cameras/camera_paths.py index e73d05ced..dc3d3dfa7 100644 --- a/nerfstudio/cameras/camera_paths.py +++ b/nerfstudio/cameras/camera_paths.py @@ -37,8 +37,8 @@ def get_interpolated_camera_path(cameras: Cameras, steps: int, order_poses: bool Returns: A new set of cameras along a path. """ - Ks = cameras.get_intrinsics_matrices().cpu().numpy() - poses = cameras.camera_to_worlds.cpu().numpy() + Ks = cameras.get_intrinsics_matrices() + poses = cameras.camera_to_worlds poses, Ks = get_interpolated_poses_many(poses, Ks, steps_per_transition=steps, order_poses=order_poses) cameras = Cameras( diff --git a/nerfstudio/cameras/camera_utils.py b/nerfstudio/cameras/camera_utils.py index 827f15d5a..c5ac2fdf2 100644 --- a/nerfstudio/cameras/camera_utils.py +++ b/nerfstudio/cameras/camera_utils.py @@ -22,13 +22,13 @@ from typing import List, Literal, Optional, Tuple import numpy as np import torch from jaxtyping import Float -from numpy.typing import ArrayLike +from numpy.typing import NDArray from torch import Tensor _EPS = np.finfo(float).eps * 4.0 -def unit_vector(data: ArrayLike, axis: Optional[int] = None) -> np.ndarray: +def unit_vector(data: NDArray, axis: Optional[int] = None) -> np.ndarray: """Return ndarray normalized by length, i.e. Euclidean norm, along axis. Args: @@ -47,7 +47,7 @@ def unit_vector(data: ArrayLike, axis: Optional[int] = None) -> np.ndarray: return data -def quaternion_from_matrix(matrix: ArrayLike, isprecise: bool = False) -> np.ndarray: +def quaternion_from_matrix(matrix: NDArray, isprecise: bool = False) -> np.ndarray: """Return quaternion from rotation matrix. Args: @@ -86,14 +86,13 @@ def quaternion_from_matrix(matrix: ArrayLike, isprecise: bool = False) -> np.nda m21 = M[2, 1] m22 = M[2, 2] # symmetric matrix K - K = np.array( - [ - [m00 - m11 - m22, 0.0, 0.0, 0.0], - [m01 + m10, m11 - m00 - m22, 0.0, 0.0], - [m02 + m20, m12 + m21, m22 - m00 - m11, 0.0], - [m21 - m12, m02 - m20, m10 - m01, m00 + m11 + m22], - ] - ) + K = [ + [m00 - m11 - m22, 0.0, 0.0, 0.0], + [m01 + m10, m11 - m00 - m22, 0.0, 0.0], + [m02 + m20, m12 + m21, m22 - m00 - m11, 0.0], + [m21 - m12, m02 - m20, m10 - m01, m00 + m11 + m22], + ] + K = np.array(K) K /= 3.0 # quaternion is eigenvector of K that corresponds to largest eigenvalue w, V = np.linalg.eigh(K) @@ -104,7 +103,7 @@ def quaternion_from_matrix(matrix: ArrayLike, isprecise: bool = False) -> np.nda def quaternion_slerp( - quat0: ArrayLike, quat1: ArrayLike, fraction: float, spin: int = 0, shortestpath: bool = True + quat0: NDArray, quat1: NDArray, fraction: float, spin: int = 0, shortestpath: bool = True ) -> np.ndarray: """Return spherical linear interpolation between two quaternions. Args: @@ -139,7 +138,7 @@ def quaternion_slerp( return q0 -def quaternion_matrix(quaternion: ArrayLike) -> np.ndarray: +def quaternion_matrix(quaternion: NDArray) -> np.ndarray: """Return homogeneous rotation matrix from quaternion. Args: @@ -161,7 +160,7 @@ def quaternion_matrix(quaternion: ArrayLike) -> np.ndarray: ) -def get_interpolated_poses(pose_a: ArrayLike, pose_b: ArrayLike, steps: int = 10) -> List[float]: +def get_interpolated_poses(pose_a: NDArray, pose_b: NDArray, steps: int = 10) -> List[float]: """Return interpolation of poses with specified number of steps. Args: pose_a: first pose @@ -185,7 +184,9 @@ def get_interpolated_poses(pose_a: ArrayLike, pose_b: ArrayLike, steps: int = 10 return poses_ab -def get_interpolated_k(k_a, k_b, steps: int = 10) -> Float[Tensor, "3 4"]: +def get_interpolated_k( + k_a: Float[Tensor, "3 3"], k_b: Float[Tensor, "3 3"], steps: int = 10 +) -> List[Float[Tensor, "3 4"]]: """ Returns interpolated path between two camera poses with specified number of steps. @@ -193,8 +194,11 @@ def get_interpolated_k(k_a, k_b, steps: int = 10) -> Float[Tensor, "3 4"]: k_a: camera matrix 1 k_b: camera matrix 2 steps: number of steps the interpolated pose path should contain + + Returns: + List of interpolated camera poses """ - Ks = [] + Ks: List[Float[Tensor, "3 3"]] = [] ts = np.linspace(0, 1, steps) for t in ts: new_k = k_a * (1.0 - t) + k_b * t @@ -220,20 +224,20 @@ def get_ordered_poses_and_k( poses_num = len(poses) - ordered_poses = np.expand_dims(poses[0], 0) - ordered_ks = np.expand_dims(Ks[0], 0) + ordered_poses = torch.unsqueeze(poses[0], 0) + ordered_ks = torch.unsqueeze(Ks[0], 0) # remove the first pose from poses - poses = np.delete(poses, 0, axis=0) - Ks = np.delete(Ks, 0, axis=0) + poses = poses[1:] + Ks = Ks[1:] for _ in range(poses_num - 1): - distances = np.linalg.norm(ordered_poses[-1][:, 3] - poses[:, :, 3], axis=1) - idx = np.argmin(distances) - ordered_poses = np.concatenate((ordered_poses, np.expand_dims(poses[idx], 0)), axis=0) - ordered_ks = np.concatenate((ordered_ks, np.expand_dims(Ks[idx], 0)), axis=0) - poses = np.delete(poses, idx, axis=0) - Ks = np.delete(Ks, idx, axis=0) + distances = torch.norm(ordered_poses[-1][:, 3] - poses[:, :, 3], dim=1) + idx = torch.argmin(distances) + ordered_poses = torch.cat((ordered_poses, torch.unsqueeze(poses[idx], 0)), dim=0) + ordered_ks = torch.cat((ordered_ks, torch.unsqueeze(Ks[idx], 0)), dim=0) + poses = torch.cat((poses[0:idx], poses[idx + 1 :]), dim=0) + Ks = torch.cat((Ks[0:idx], Ks[idx + 1 :]), dim=0) return ordered_poses, ordered_ks @@ -262,14 +266,14 @@ def get_interpolated_poses_many( poses, Ks = get_ordered_poses_and_k(poses, Ks) for idx in range(poses.shape[0] - 1): - pose_a = poses[idx] - pose_b = poses[idx + 1] + pose_a = poses[idx].cpu().numpy() + pose_b = poses[idx + 1].cpu().numpy() poses_ab = get_interpolated_poses(pose_a, pose_b, steps=steps_per_transition) traj += poses_ab k_interp += get_interpolated_k(Ks[idx], Ks[idx + 1], steps=steps_per_transition) traj = np.stack(traj, axis=0) - k_interp = np.stack(k_interp, axis=0) + k_interp = torch.stack(k_interp, dim=0) return torch.tensor(traj, dtype=torch.float32), torch.tensor(k_interp, dtype=torch.float32) @@ -565,7 +569,7 @@ def auto_orient_and_center_poses( transform = torch.cat([eigvec, eigvec @ -translation[..., None]], dim=-1) oriented_poses = transform @ poses - if oriented_poses.mean(axis=0)[2, 1] < 0: + if oriented_poses.mean(dim=0)[2, 1] < 0: oriented_poses[:, 1:3] = -1 * oriented_poses[:, 1:3] elif method in ("up", "vertical"): up = torch.mean(poses[:, :3, 1], dim=0) diff --git a/nerfstudio/cameras/cameras.py b/nerfstudio/cameras/cameras.py index cb4e238a7..457b78f7b 100644 --- a/nerfstudio/cameras/cameras.py +++ b/nerfstudio/cameras/cameras.py @@ -100,13 +100,11 @@ class Cameras(TensorDataclass): width: Optional[Union[Shaped[Tensor, "*batch_ws 1"], int]] = None, height: Optional[Union[Shaped[Tensor, "*batch_hs 1"], int]] = None, distortion_params: Optional[Float[Tensor, "*batch_dist_params 6"]] = None, - camera_type: Optional[ - Union[ - Int[Tensor, "*batch_cam_types 1"], - int, - List[CameraType], - CameraType, - ] + camera_type: Union[ + Int[Tensor, "*batch_cam_types 1"], + int, + List[CameraType], + CameraType, ] = CameraType.PERSPECTIVE, times: Optional[Float[Tensor, "num_cameras"]] = None, metadata: Optional[Dict] = None, @@ -297,8 +295,8 @@ class Cameras(TensorDataclass): Grid of image coordinates. """ if index is None: - image_height = torch.max(self.image_height.view(-1)) - image_width = torch.max(self.image_width.view(-1)) + image_height = torch.max(self.image_height.view(-1)).item() + image_width = torch.max(self.image_width.view(-1)).item() image_coords = torch.meshgrid(torch.arange(image_height), torch.arange(image_width), indexing="ij") image_coords = torch.stack(image_coords, dim=-1) + pixel_offset # stored as (y, x) coordinates else: @@ -748,6 +746,9 @@ class Cameras(TensorDataclass): A JSON representation of the camera """ flattened = self.flatten() + times = flattened[camera_idx].times + if times is not None: + times = times.item() json_ = { "type": "PinholeCamera", "cx": flattened[camera_idx].cx.item(), @@ -756,7 +757,7 @@ class Cameras(TensorDataclass): "fy": flattened[camera_idx].fy.item(), "camera_to_world": self.camera_to_worlds[camera_idx].tolist(), "camera_index": camera_idx, - "times": flattened[camera_idx].times.item() if self.times is not None else None, + "times": times, } if image is not None: image_uint8 = (image * 255).detach().type(torch.uint8) diff --git a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py index 9361300ce..1bbf0b455 100644 --- a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py +++ b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py @@ -17,7 +17,7 @@ from __future__ import annotations import math from dataclasses import dataclass, field -from pathlib import Path, PurePath +from pathlib import Path from typing import Literal, Optional, Type import numpy as np @@ -109,7 +109,7 @@ class Nerfstudio(DataParser): distort = [] for frame in meta["frames"]: - filepath = PurePath(frame["file_path"]) + filepath = Path(frame["file_path"]) fname = self._get_fname(filepath, data_dir) if not fname.exists(): num_skipped_image_filenames += 1 @@ -148,7 +148,7 @@ class Nerfstudio(DataParser): image_filenames.append(fname) poses.append(np.array(frame["transform_matrix"])) if "mask_path" in frame: - mask_filepath = PurePath(frame["mask_path"]) + mask_filepath = Path(frame["mask_path"]) mask_fname = self._get_fname( mask_filepath, data_dir, @@ -157,7 +157,7 @@ class Nerfstudio(DataParser): mask_filenames.append(mask_fname) if "depth_file_path" in frame: - depth_filepath = PurePath(frame["depth_file_path"]) + depth_filepath = Path(frame["depth_file_path"]) depth_fname = self._get_fname(depth_filepath, data_dir, downsample_folder_prefix="depths_") depth_filenames.append(depth_fname) @@ -185,7 +185,7 @@ class Nerfstudio(DataParser): has_split_files_spec = any(f"{split}_filenames" in meta for split in ("train", "val", "test")) if f"{split}_filenames" in meta: # Validate split first - split_filenames = set(self._get_fname(PurePath(x), data_dir) for x in meta[f"{split}_filenames"]) + split_filenames = set(self._get_fname(Path(x), data_dir) for x in meta[f"{split}_filenames"]) unmatched_filenames = split_filenames.difference(image_filenames) if unmatched_filenames: raise RuntimeError(f"Some filenames for split {split} were not found: {unmatched_filenames}.") @@ -238,7 +238,9 @@ class Nerfstudio(DataParser): image_filenames = [image_filenames[i] for i in indices] mask_filenames = [mask_filenames[i] for i in indices] if len(mask_filenames) > 0 else [] depth_filenames = [depth_filenames[i] for i in indices] if len(depth_filenames) > 0 else [] - poses = poses[indices] + + idx_tensor = torch.tensor(indices, dtype=torch.long) + poses = poses[idx_tensor] # in x,y,z order # assumes that the scene is centered at the origin @@ -254,7 +256,6 @@ class Nerfstudio(DataParser): else: camera_type = CameraType.PERSPECTIVE - idx_tensor = torch.tensor(indices, dtype=torch.long) fx = float(meta["fl_x"]) if fx_fixed else torch.tensor(fx, dtype=torch.float32)[idx_tensor] fy = float(meta["fl_y"]) if fy_fixed else torch.tensor(fy, dtype=torch.float32)[idx_tensor] cx = float(meta["cx"]) if cx_fixed else torch.tensor(cx, dtype=torch.float32)[idx_tensor] @@ -311,7 +312,7 @@ class Nerfstudio(DataParser): ) return dataparser_outputs - def _get_fname(self, filepath: PurePath, data_dir: PurePath, downsample_folder_prefix="images_") -> Path: + def _get_fname(self, filepath: Path, data_dir: Path, downsample_folder_prefix="images_") -> Path: """Get the filename of the image file. downsample_folder_prefix can be used to point to auxiliary image data, e.g. masks diff --git a/nerfstudio/fields/nerfplayer_nerfacto_field.py b/nerfstudio/fields/nerfplayer_nerfacto_field.py index 9ce2a02d2..06b3df1b4 100644 --- a/nerfstudio/fields/nerfplayer_nerfacto_field.py +++ b/nerfstudio/fields/nerfplayer_nerfacto_field.py @@ -136,6 +136,7 @@ class TemporalHashMLPDensityField(Field): else: positions = SceneBox.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) positions_flat = positions.view(-1, 3) + assert ray_samples.times is not None time_flat = ray_samples.times.reshape(-1, 1) x = self.encoding(positions_flat, time_flat).to(positions) density_before_activation = self.linear(x).view(*ray_samples.frustums.shape, -1) @@ -233,7 +234,7 @@ class NerfplayerNerfactoField(Field): num_levels=num_levels, level_dim=features_per_level, log2_hashmap_size=log2_hashmap_size, - desired_resolution=1024 * (self.aabb.max() - self.aabb.min()), + desired_resolution=int(1024 * (self.aabb.max().item() - self.aabb.min().item())), ) self.mlp_base_decode = tcnn.Network( n_input_dims=num_levels * features_per_level, diff --git a/nerfstudio/scripts/exporter.py b/nerfstudio/scripts/exporter.py index 1bb2055ec..09e0de716 100644 --- a/nerfstudio/scripts/exporter.py +++ b/nerfstudio/scripts/exporter.py @@ -440,5 +440,7 @@ def entrypoint(): if __name__ == "__main__": entrypoint() -# For sphinx docs -get_parser_fn = lambda: tyro.extras.get_parser(Commands) # noqa + +def get_parser_fn(): + """Get the parser function for the sphinx docs.""" + return tyro.extras.get_parser(Commands) # noqa diff --git a/nerfstudio/scripts/render.py b/nerfstudio/scripts/render.py index afe33ccd9..d97d6b6c7 100644 --- a/nerfstudio/scripts/render.py +++ b/nerfstudio/scripts/render.py @@ -448,5 +448,7 @@ def entrypoint(): if __name__ == "__main__": entrypoint() -# For sphinx docs -get_parser_fn = lambda: tyro.extras.get_parser(Commands) # noqa + +def get_parser_fn(): + """Get the parser function for the sphinx docs.""" + return tyro.extras.get_parser(Commands) # noqa