Merge branch 'main' into gina/auto_camera_render

docs/nerfology/methods/splatw.md

@@ -1,9 +1,10 @@
 # Splatfacto in the Wild
 
-This is the implementation of [Splatfacto in the Wild: A Nerfstudio Implementation of Gaussian Splatting for Unconstrained Photo Collections](https://kevinxu02.github.io/gsw.github.io/). The official code can be found [here](https://github.com/KevinXu02/splatfacto-w).
+This is the implementation of [Splatfacto in the Wild: A Nerfstudio Implementation of Gaussian Splatting for Unconstrained Photo Collections](https://kevinxu02.github.io/splatfactow). The official code can be found [here](https://github.com/KevinXu02/splatfacto-w).
 
 <video id="teaser" muted autoplay playsinline loop controls width="100%">
-    <source id="mp4" src="https://github.com/KevinXu02/gsw.github.io/blob/main/static/videos/interp_fountain2.mp4" type="video/mp4">
+    <source id="mp4" src="https://github.com/KevinXu02/splatfactow/blob/main/static/videos/interp_fountain2.mp4" type="video/mp4">
+
 </video>
 
 ## Installation
@@ -28,14 +29,23 @@ ns-download-data phototourism --capture-name <capture_name>
 ```
 
 ## Running Splafacto-w
-To train with it, download the train/test tsv file from the bottom of [nerf-w](https://nerf-w.github.io/) and put it under the data folder (or copy them from `.\splatfacto-w\dataset_split`). For instance, for Brandenburg Gate the path would be `splatfacto-w\data\brandenburg_gate\brandenburg.tsv`.
+To train with it, download the train/test tsv file from the bottom of [nerf-w](https://nerf-w.github.io/) and put it under the data folder (or copy them from `./splatfacto-w/dataset_split`). For instance, for Brandenburg Gate the path would be `your-data-folder/brandenburg_gate/brandenburg.tsv`. You should have the following structure in your data folder:
+```
+|---brandenburg_gate
+|   |---dense
+|   |   |---images
+|   |   |---sparse
+|   |   |---stereo
+|   |---brandenburg.tsv
+```
+
 Then, run the command:
 ```
 ns-train splatfacto-w --data [PATH]
 ```
 
-If you want to train datasets without nerf-w's train/test split or your own datasets, you can run the following command:
+If you want to train datasets without nerf-w's train/test split or your own datasets, we provided a light-weight version of the method for general cases. To train with it, you can run the following command
 ```
-ns-train splatfacto-w --data [PATH] [dataparser]
+ns-train splatfacto-w-light --data [PATH] [dataparser]
 ```
 For phototourism, the `dataparser` should be `colmap` and you need to change the colmap path through the CLI because phototourism dataparser does not load 3D points.

nerfstudio/cameras/camera_utils.py

@@ -54,7 +54,7 @@ def quaternion_from_matrix(matrix: NDArray, isprecise: bool = False) -> np.ndarr
         matrix: rotation matrix to obtain quaternion
         isprecise: if True, input matrix is assumed to be precise rotation matrix and a faster algorithm is used.
     """
-    M = np.array(matrix, dtype=np.float64, copy=False)[:4, :4]
+    M = np.array(matrix, dtype=np.float64, copy=True)[:4, :4]
     if isprecise:
         q = np.empty((4,))
         t = np.trace(M)

nerfstudio/data/pixel_samplers.py

@@ -337,7 +337,7 @@ class PixelSampler:
         collated_batch = {
             key: value[c, y, x]
             for key, value in batch.items()
-            if key != "image_idx" and key != "image" and key != "mask" and key != "depth_image" and value is not None
+            if key not in ("image_idx", "image", "mask", "depth_image") and value is not None
         }
 
         collated_batch["image"] = torch.cat(all_images, dim=0)

nerfstudio/data/utils/nerfstudio_collate.py

@@ -101,7 +101,7 @@ def nerfstudio_collate(batch: Any, extra_mappings: Union[Dict[type, Callable], N
             storage = elem.storage()._new_shared(numel, device=elem.device)
             out = elem.new(storage).resize_(len(batch), *list(elem.size()))
         return torch.stack(batch, 0, out=out)
-    elif elem_type.__module__ == "numpy" and elem_type.__name__ != "str_" and elem_type.__name__ != "string_":
+    elif elem_type.__module__ == "numpy" and elem_type.__name__ not in ("str_", "string_"):
         if elem_type.__name__ in ("ndarray", "memmap"):
             # array of string classes and object
             if np_str_obj_array_pattern.search(elem.dtype.str) is not None:

nerfstudio/viewer/utils.py

@@ -42,6 +42,8 @@ class CameraState:
     """Type of camera to render."""
     time: float = 0.0
     """The rendering time of the camera state."""
+    idx: int = 0
+    """The index of the current camera."""
 
 
 def get_camera(
@@ -78,6 +80,7 @@ def get_camera(
         camera_type=camera_state.camera_type,
         camera_to_worlds=camera_state.c2w.to(torch.float32)[None, ...],
         times=torch.tensor([camera_state.time], dtype=torch.float32),
+        metadata={"cam_idx": camera_state.idx},
     )
     return camera
 

nerfstudio/viewer/viewer.py

@@ -106,6 +106,8 @@ class Viewer:
         self.train_btn_state: Literal["training", "paused", "completed"] = "training"
         self._prev_train_state: Literal["training", "paused", "completed"] = "training"
         self.last_move_time = 0
+        # track the camera index that last being clicked
+        self.current_camera_idx = 0
 
         self.viser_server = viser.ViserServer(host=config.websocket_host, port=websocket_port)
         # Set the name of the URL either to the share link if available, or the localhost
@@ -326,6 +328,7 @@ class Viewer:
                 else CameraType.EQUIRECTANGULAR
                 if camera_type == "Equirectangular"
                 else assert_never(camera_type),
+                idx=self.current_camera_idx,
             )
         else:
             camera_state = CameraState(
@@ -333,6 +336,7 @@ class Viewer:
                 aspect=client.camera.aspect,
                 c2w=c2w,
                 camera_type=CameraType.PERSPECTIVE,
+                idx=self.current_camera_idx,
             )
         return camera_state
 
@@ -463,11 +467,16 @@ class Viewer:
                 position=c2w[:3, 3] * VISER_NERFSTUDIO_SCALE_RATIO,
             )
 
-            @camera_handle.on_click
-            def _(event: viser.SceneNodePointerEvent[viser.CameraFrustumHandle]) -> None:
-                with event.client.atomic():
-                    event.client.camera.position = event.target.position
-                    event.client.camera.wxyz = event.target.wxyz
+            def create_on_click_callback(capture_idx):
+                def on_click_callback(event: viser.SceneNodePointerEvent[viser.CameraFrustumHandle]) -> None:
+                    with event.client.atomic():
+                        event.client.camera.position = event.target.position
+                        event.client.camera.wxyz = event.target.wxyz
+                        self.current_camera_idx = capture_idx
+
+                return on_click_callback
+
+            camera_handle.on_click(create_on_click_callback(idx))
 
             self.camera_handles[idx] = camera_handle
             self.original_c2w[idx] = c2w

pyproject.toml

@@ -33,7 +33,7 @@ dependencies = [
     "msgpack_numpy>=0.4.8",
     "nerfacc==0.5.2",
     "open3d>=0.16.0",
-    "opencv-python==4.10.0.84",
+    "opencv-python-headless==4.10.0.84",
     "Pillow>=10.3.0",
     "plotly>=5.7.0",
     "protobuf<=3.20.3,!=3.20.0",