Improved interpolate render (#1911)

* [WIP] Speed is adjusted depending on camera distance

* [WIP] Speed is adjusted depending on camera distance

* Speed is adjusted depending on camera distance

* Constant time between poses and ordered poses

* Removed old speed adjust

* remove prints

* renamed variables

* added launch.json back

* added back blank lines

* Trying to solve blank line git  issue in launch.json

* Added docstring and type annotations

---------

Co-authored-by: jperezh <jperezh@arquimea.com>
Co-authored-by: Matthew Tancik <tancik@berkeley.edu>

nerfstudio/cameras/camera_paths.py

@@ -27,7 +27,7 @@ from nerfstudio.cameras.cameras import Cameras, CameraType
 from nerfstudio.viewer.server.utils import three_js_perspective_camera_focal_length
 
 
-def get_interpolated_camera_path(cameras: Cameras, steps: int) -> Cameras:
+def get_interpolated_camera_path(cameras: Cameras, steps: int, order_poses: bool) -> Cameras:
     """Generate a camera path between two cameras.
 
     Args:
@@ -39,7 +39,7 @@ def get_interpolated_camera_path(cameras: Cameras, steps: int) -> Cameras:
     """
     Ks = cameras.get_intrinsics_matrices().cpu().numpy()
     poses = cameras.camera_to_worlds.cpu().numpy()
-    poses, Ks = get_interpolated_poses_many(poses, Ks, steps_per_transition=steps)
+    poses, Ks = get_interpolated_poses_many(poses, Ks, steps_per_transition=steps, order_poses=order_poses)
 
     cameras = Cameras(fx=Ks[:, 0, 0], fy=Ks[:, 1, 1], cx=Ks[0, 0, 2], cy=Ks[0, 1, 2], camera_to_worlds=poses)
     return cameras

nerfstudio/cameras/camera_utils.py

@@ -201,10 +201,47 @@ def get_interpolated_k(k_a, k_b, steps: int = 10) -> TensorType[3, 4]:
     return Ks
 
 
+def get_ordered_poses_and_k(
+    poses: TensorType["num_poses", 3, 4],
+    Ks: TensorType["num_poses", 3, 3],
+) -> Tuple[TensorType["num_poses", 3, 4], TensorType["num_poses", 3, 3]]:
+    """
+    Returns ordered poses and intrinsics by euclidian distance between poses.
+
+    Args:
+        poses: list of camera poses
+        Ks: list of camera intrinsics
+
+    Returns:
+        tuple of ordered poses and intrinsics
+
+    """
+
+    poses_num = len(poses)
+
+    ordered_poses = np.expand_dims(poses[0], 0)
+    ordered_ks = np.expand_dims(Ks[0], 0)
+
+    # remove the first pose from poses
+    poses = np.delete(poses, 0, axis=0)
+    Ks = np.delete(Ks, 0, axis=0)
+
+    for _ in range(poses_num - 1):
+        distances = np.linalg.norm(ordered_poses[-1][:, 3] - poses[:, :, 3], axis=1)
+        idx = np.argmin(distances)
+        ordered_poses = np.concatenate((ordered_poses, np.expand_dims(poses[idx], 0)), axis=0)
+        ordered_ks = np.concatenate((ordered_ks, np.expand_dims(Ks[idx], 0)), axis=0)
+        poses = np.delete(poses, idx, axis=0)
+        Ks = np.delete(Ks, idx, axis=0)
+
+    return ordered_poses, ordered_ks
+
+
 def get_interpolated_poses_many(
     poses: TensorType["num_poses", 3, 4],
     Ks: TensorType["num_poses", 3, 3],
-    steps_per_transition=10,
+    steps_per_transition: int = 10,
+    order_poses: bool = False,
 ) -> Tuple[TensorType["num_poses", 3, 4], TensorType["num_poses", 3, 3]]:
     """Return interpolated poses for many camera poses.
 
@@ -212,12 +249,17 @@ def get_interpolated_poses_many(
         poses: list of camera poses
         Ks: list of camera intrinsics
         steps_per_transition: number of steps per transition
+        order_poses: whether to order poses by euclidian distance
 
     Returns:
         tuple of new poses and intrinsics
     """
     traj = []
     k_interp = []
+
+    if order_poses:
+        poses, Ks = get_ordered_poses_and_k(poses, Ks)
+
     for idx in range(poses.shape[0] - 1):
         pose_a = poses[idx]
         pose_b = poses[idx + 1]

nerfstudio/scripts/render.py

@@ -310,6 +310,14 @@ class RenderTrajectory:
     """How to save output data."""
     interpolation_steps: int = 10
     """Number of interpolation steps between eval dataset cameras."""
+    order_poses: bool = False
+    """Whether to order camera poses by proximity (only for interpolate trajectory)."""
+    adjust_frame_rate: bool = False
+    """Whether to adjust the frame rate of the output video (only for interpolate trajectory)."""
+    frame_rate: int = 24
+    """Frame rate of the output video (only for interpolate trajectory)."""
+    seconds_between_poses: int = 3
+    """Seconds between poses for interpolated camera paths."""
     eval_num_rays_per_chunk: Optional[int] = None
     """Specifies number of rays per chunk during eval."""
     colormap_options: colormaps.ColormapOptions = colormaps.ColormapOptions()
@@ -350,8 +358,13 @@ class RenderTrajectory:
             camera_path = get_path_from_json(camera_path)
         elif self.traj == "interpolate":
             camera_type = CameraType.PERSPECTIVE
+            if self.adjust_frame_rate:
+                self.interpolation_steps = self.seconds_between_poses * self.frame_rate
+                seconds = self.seconds_between_poses * len(pipeline.datamanager.eval_dataloader.cameras)
             camera_path = get_interpolated_camera_path(
-                cameras=pipeline.datamanager.eval_dataloader.cameras, steps=self.interpolation_steps
+                cameras=pipeline.datamanager.eval_dataloader.cameras,
+                steps=self.interpolation_steps,
+                order_poses=self.order_poses,
             )
         else:
             assert_never(self.traj)