From bc2651963c8425bcf379f80048e5de288ffa2491 Mon Sep 17 00:00:00 2001 From: Fernando Rivas-Manzaneque <98962882+Frivas97@users.noreply.github.com> Date: Fri, 12 May 2023 23:19:13 +0100 Subject: [PATCH] Improved interpolate render (#1911) * [WIP] Speed is adjusted depending on camera distance * [WIP] Speed is adjusted depending on camera distance * Speed is adjusted depending on camera distance * Constant time between poses and ordered poses * Removed old speed adjust * remove prints * renamed variables * added launch.json back * added back blank lines * Trying to solve blank line git issue in launch.json * Added docstring and type annotations --------- Co-authored-by: jperezh Co-authored-by: Matthew Tancik --- nerfstudio/cameras/camera_paths.py | 4 +-- nerfstudio/cameras/camera_utils.py | 44 +++++++++++++++++++++++++++++- nerfstudio/scripts/render.py | 15 +++++++++- 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/nerfstudio/cameras/camera_paths.py b/nerfstudio/cameras/camera_paths.py index 9754a2bbf..f8117aeb2 100644 --- a/nerfstudio/cameras/camera_paths.py +++ b/nerfstudio/cameras/camera_paths.py @@ -27,7 +27,7 @@ from nerfstudio.cameras.cameras import Cameras, CameraType from nerfstudio.viewer.server.utils import three_js_perspective_camera_focal_length -def get_interpolated_camera_path(cameras: Cameras, steps: int) -> Cameras: +def get_interpolated_camera_path(cameras: Cameras, steps: int, order_poses: bool) -> Cameras: """Generate a camera path between two cameras. Args: @@ -39,7 +39,7 @@ def get_interpolated_camera_path(cameras: Cameras, steps: int) -> Cameras: """ Ks = cameras.get_intrinsics_matrices().cpu().numpy() poses = cameras.camera_to_worlds.cpu().numpy() - poses, Ks = get_interpolated_poses_many(poses, Ks, steps_per_transition=steps) + poses, Ks = get_interpolated_poses_many(poses, Ks, steps_per_transition=steps, order_poses=order_poses) cameras = Cameras(fx=Ks[:, 0, 0], fy=Ks[:, 1, 1], cx=Ks[0, 0, 2], cy=Ks[0, 1, 2], camera_to_worlds=poses) return cameras diff --git a/nerfstudio/cameras/camera_utils.py b/nerfstudio/cameras/camera_utils.py index 7c2364929..2b79ad673 100644 --- a/nerfstudio/cameras/camera_utils.py +++ b/nerfstudio/cameras/camera_utils.py @@ -201,10 +201,47 @@ def get_interpolated_k(k_a, k_b, steps: int = 10) -> TensorType[3, 4]: return Ks +def get_ordered_poses_and_k( + poses: TensorType["num_poses", 3, 4], + Ks: TensorType["num_poses", 3, 3], +) -> Tuple[TensorType["num_poses", 3, 4], TensorType["num_poses", 3, 3]]: + """ + Returns ordered poses and intrinsics by euclidian distance between poses. + + Args: + poses: list of camera poses + Ks: list of camera intrinsics + + Returns: + tuple of ordered poses and intrinsics + + """ + + poses_num = len(poses) + + ordered_poses = np.expand_dims(poses[0], 0) + ordered_ks = np.expand_dims(Ks[0], 0) + + # remove the first pose from poses + poses = np.delete(poses, 0, axis=0) + Ks = np.delete(Ks, 0, axis=0) + + for _ in range(poses_num - 1): + distances = np.linalg.norm(ordered_poses[-1][:, 3] - poses[:, :, 3], axis=1) + idx = np.argmin(distances) + ordered_poses = np.concatenate((ordered_poses, np.expand_dims(poses[idx], 0)), axis=0) + ordered_ks = np.concatenate((ordered_ks, np.expand_dims(Ks[idx], 0)), axis=0) + poses = np.delete(poses, idx, axis=0) + Ks = np.delete(Ks, idx, axis=0) + + return ordered_poses, ordered_ks + + def get_interpolated_poses_many( poses: TensorType["num_poses", 3, 4], Ks: TensorType["num_poses", 3, 3], - steps_per_transition=10, + steps_per_transition: int = 10, + order_poses: bool = False, ) -> Tuple[TensorType["num_poses", 3, 4], TensorType["num_poses", 3, 3]]: """Return interpolated poses for many camera poses. @@ -212,12 +249,17 @@ def get_interpolated_poses_many( poses: list of camera poses Ks: list of camera intrinsics steps_per_transition: number of steps per transition + order_poses: whether to order poses by euclidian distance Returns: tuple of new poses and intrinsics """ traj = [] k_interp = [] + + if order_poses: + poses, Ks = get_ordered_poses_and_k(poses, Ks) + for idx in range(poses.shape[0] - 1): pose_a = poses[idx] pose_b = poses[idx + 1] diff --git a/nerfstudio/scripts/render.py b/nerfstudio/scripts/render.py index 628be9d2e..50790a4b7 100644 --- a/nerfstudio/scripts/render.py +++ b/nerfstudio/scripts/render.py @@ -310,6 +310,14 @@ class RenderTrajectory: """How to save output data.""" interpolation_steps: int = 10 """Number of interpolation steps between eval dataset cameras.""" + order_poses: bool = False + """Whether to order camera poses by proximity (only for interpolate trajectory).""" + adjust_frame_rate: bool = False + """Whether to adjust the frame rate of the output video (only for interpolate trajectory).""" + frame_rate: int = 24 + """Frame rate of the output video (only for interpolate trajectory).""" + seconds_between_poses: int = 3 + """Seconds between poses for interpolated camera paths.""" eval_num_rays_per_chunk: Optional[int] = None """Specifies number of rays per chunk during eval.""" colormap_options: colormaps.ColormapOptions = colormaps.ColormapOptions() @@ -350,8 +358,13 @@ class RenderTrajectory: camera_path = get_path_from_json(camera_path) elif self.traj == "interpolate": camera_type = CameraType.PERSPECTIVE + if self.adjust_frame_rate: + self.interpolation_steps = self.seconds_between_poses * self.frame_rate + seconds = self.seconds_between_poses * len(pipeline.datamanager.eval_dataloader.cameras) camera_path = get_interpolated_camera_path( - cameras=pipeline.datamanager.eval_dataloader.cameras, steps=self.interpolation_steps + cameras=pipeline.datamanager.eval_dataloader.cameras, + steps=self.interpolation_steps, + order_poses=self.order_poses, ) else: assert_never(self.traj)