diff --git a/docs/nerfology/methods/splatw.md b/docs/nerfology/methods/splatw.md index 85b08af91..21c772a39 100644 --- a/docs/nerfology/methods/splatw.md +++ b/docs/nerfology/methods/splatw.md @@ -1,9 +1,10 @@ # Splatfacto in the Wild -This is the implementation of [Splatfacto in the Wild: A Nerfstudio Implementation of Gaussian Splatting for Unconstrained Photo Collections](https://kevinxu02.github.io/gsw.github.io/). The official code can be found [here](https://github.com/KevinXu02/splatfacto-w). +This is the implementation of [Splatfacto in the Wild: A Nerfstudio Implementation of Gaussian Splatting for Unconstrained Photo Collections](https://kevinxu02.github.io/splatfactow). The official code can be found [here](https://github.com/KevinXu02/splatfacto-w). ## Installation @@ -28,14 +29,23 @@ ns-download-data phototourism --capture-name ``` ## Running Splafacto-w -To train with it, download the train/test tsv file from the bottom of [nerf-w](https://nerf-w.github.io/) and put it under the data folder (or copy them from `.\splatfacto-w\dataset_split`). For instance, for Brandenburg Gate the path would be `splatfacto-w\data\brandenburg_gate\brandenburg.tsv`. +To train with it, download the train/test tsv file from the bottom of [nerf-w](https://nerf-w.github.io/) and put it under the data folder (or copy them from `./splatfacto-w/dataset_split`). For instance, for Brandenburg Gate the path would be `your-data-folder/brandenburg_gate/brandenburg.tsv`. You should have the following structure in your data folder: +``` +|---brandenburg_gate +| |---dense +| | |---images +| | |---sparse +| | |---stereo +| |---brandenburg.tsv +``` + Then, run the command: ``` ns-train splatfacto-w --data [PATH] ``` -If you want to train datasets without nerf-w's train/test split or your own datasets, you can run the following command: +If you want to train datasets without nerf-w's train/test split or your own datasets, we provided a light-weight version of the method for general cases. To train with it, you can run the following command ``` -ns-train splatfacto-w --data [PATH] [dataparser] +ns-train splatfacto-w-light --data [PATH] [dataparser] ``` For phototourism, the `dataparser` should be `colmap` and you need to change the colmap path through the CLI because phototourism dataparser does not load 3D points. \ No newline at end of file diff --git a/nerfstudio/cameras/camera_utils.py b/nerfstudio/cameras/camera_utils.py index 185a852a2..3e8785033 100644 --- a/nerfstudio/cameras/camera_utils.py +++ b/nerfstudio/cameras/camera_utils.py @@ -54,7 +54,7 @@ def quaternion_from_matrix(matrix: NDArray, isprecise: bool = False) -> np.ndarr matrix: rotation matrix to obtain quaternion isprecise: if True, input matrix is assumed to be precise rotation matrix and a faster algorithm is used. """ - M = np.array(matrix, dtype=np.float64, copy=False)[:4, :4] + M = np.array(matrix, dtype=np.float64, copy=True)[:4, :4] if isprecise: q = np.empty((4,)) t = np.trace(M) diff --git a/nerfstudio/data/pixel_samplers.py b/nerfstudio/data/pixel_samplers.py index ad11ee409..080b4016a 100644 --- a/nerfstudio/data/pixel_samplers.py +++ b/nerfstudio/data/pixel_samplers.py @@ -337,7 +337,7 @@ class PixelSampler: collated_batch = { key: value[c, y, x] for key, value in batch.items() - if key != "image_idx" and key != "image" and key != "mask" and key != "depth_image" and value is not None + if key not in ("image_idx", "image", "mask", "depth_image") and value is not None } collated_batch["image"] = torch.cat(all_images, dim=0) diff --git a/nerfstudio/data/utils/nerfstudio_collate.py b/nerfstudio/data/utils/nerfstudio_collate.py index 8c8a633fb..b5b391f54 100644 --- a/nerfstudio/data/utils/nerfstudio_collate.py +++ b/nerfstudio/data/utils/nerfstudio_collate.py @@ -101,7 +101,7 @@ def nerfstudio_collate(batch: Any, extra_mappings: Union[Dict[type, Callable], N storage = elem.storage()._new_shared(numel, device=elem.device) out = elem.new(storage).resize_(len(batch), *list(elem.size())) return torch.stack(batch, 0, out=out) - elif elem_type.__module__ == "numpy" and elem_type.__name__ != "str_" and elem_type.__name__ != "string_": + elif elem_type.__module__ == "numpy" and elem_type.__name__ not in ("str_", "string_"): if elem_type.__name__ in ("ndarray", "memmap"): # array of string classes and object if np_str_obj_array_pattern.search(elem.dtype.str) is not None: diff --git a/nerfstudio/viewer/utils.py b/nerfstudio/viewer/utils.py index 0f5ead912..2fbb5af14 100644 --- a/nerfstudio/viewer/utils.py +++ b/nerfstudio/viewer/utils.py @@ -42,6 +42,8 @@ class CameraState: """Type of camera to render.""" time: float = 0.0 """The rendering time of the camera state.""" + idx: int = 0 + """The index of the current camera.""" def get_camera( @@ -78,6 +80,7 @@ def get_camera( camera_type=camera_state.camera_type, camera_to_worlds=camera_state.c2w.to(torch.float32)[None, ...], times=torch.tensor([camera_state.time], dtype=torch.float32), + metadata={"cam_idx": camera_state.idx}, ) return camera diff --git a/nerfstudio/viewer/viewer.py b/nerfstudio/viewer/viewer.py index c854456a5..4d39a0e89 100644 --- a/nerfstudio/viewer/viewer.py +++ b/nerfstudio/viewer/viewer.py @@ -106,6 +106,8 @@ class Viewer: self.train_btn_state: Literal["training", "paused", "completed"] = "training" self._prev_train_state: Literal["training", "paused", "completed"] = "training" self.last_move_time = 0 + # track the camera index that last being clicked + self.current_camera_idx = 0 self.viser_server = viser.ViserServer(host=config.websocket_host, port=websocket_port) # Set the name of the URL either to the share link if available, or the localhost @@ -326,6 +328,7 @@ class Viewer: else CameraType.EQUIRECTANGULAR if camera_type == "Equirectangular" else assert_never(camera_type), + idx=self.current_camera_idx, ) else: camera_state = CameraState( @@ -333,6 +336,7 @@ class Viewer: aspect=client.camera.aspect, c2w=c2w, camera_type=CameraType.PERSPECTIVE, + idx=self.current_camera_idx, ) return camera_state @@ -463,11 +467,16 @@ class Viewer: position=c2w[:3, 3] * VISER_NERFSTUDIO_SCALE_RATIO, ) - @camera_handle.on_click - def _(event: viser.SceneNodePointerEvent[viser.CameraFrustumHandle]) -> None: - with event.client.atomic(): - event.client.camera.position = event.target.position - event.client.camera.wxyz = event.target.wxyz + def create_on_click_callback(capture_idx): + def on_click_callback(event: viser.SceneNodePointerEvent[viser.CameraFrustumHandle]) -> None: + with event.client.atomic(): + event.client.camera.position = event.target.position + event.client.camera.wxyz = event.target.wxyz + self.current_camera_idx = capture_idx + + return on_click_callback + + camera_handle.on_click(create_on_click_callback(idx)) self.camera_handles[idx] = camera_handle self.original_c2w[idx] = c2w diff --git a/pyproject.toml b/pyproject.toml index 499815b3e..397433cba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ "msgpack_numpy>=0.4.8", "nerfacc==0.5.2", "open3d>=0.16.0", - "opencv-python==4.10.0.84", + "opencv-python-headless==4.10.0.84", "Pillow>=10.3.0", "plotly>=5.7.0", "protobuf<=3.20.3,!=3.20.0",