From d2510aa02f964fefae26b083a72333ff99f952d2 Mon Sep 17 00:00:00 2001 From: Congrong Xu <50019703+KevinXu02@users.noreply.github.com> Date: Thu, 18 Jul 2024 15:59:55 +0800 Subject: [PATCH 1/5] Splatw doc updates (#3311) * splatw doc * fix * fix typo * update doc --------- Co-authored-by: Justin Kerr --- docs/nerfology/methods/splatw.md | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/docs/nerfology/methods/splatw.md b/docs/nerfology/methods/splatw.md index 85b08af91..21c772a39 100644 --- a/docs/nerfology/methods/splatw.md +++ b/docs/nerfology/methods/splatw.md @@ -1,9 +1,10 @@ # Splatfacto in the Wild -This is the implementation of [Splatfacto in the Wild: A Nerfstudio Implementation of Gaussian Splatting for Unconstrained Photo Collections](https://kevinxu02.github.io/gsw.github.io/). The official code can be found [here](https://github.com/KevinXu02/splatfacto-w). +This is the implementation of [Splatfacto in the Wild: A Nerfstudio Implementation of Gaussian Splatting for Unconstrained Photo Collections](https://kevinxu02.github.io/splatfactow). The official code can be found [here](https://github.com/KevinXu02/splatfacto-w). ## Installation @@ -28,14 +29,23 @@ ns-download-data phototourism --capture-name ``` ## Running Splafacto-w -To train with it, download the train/test tsv file from the bottom of [nerf-w](https://nerf-w.github.io/) and put it under the data folder (or copy them from `.\splatfacto-w\dataset_split`). For instance, for Brandenburg Gate the path would be `splatfacto-w\data\brandenburg_gate\brandenburg.tsv`. +To train with it, download the train/test tsv file from the bottom of [nerf-w](https://nerf-w.github.io/) and put it under the data folder (or copy them from `./splatfacto-w/dataset_split`). For instance, for Brandenburg Gate the path would be `your-data-folder/brandenburg_gate/brandenburg.tsv`. You should have the following structure in your data folder: +``` +|---brandenburg_gate +| |---dense +| | |---images +| | |---sparse +| | |---stereo +| |---brandenburg.tsv +``` + Then, run the command: ``` ns-train splatfacto-w --data [PATH] ``` -If you want to train datasets without nerf-w's train/test split or your own datasets, you can run the following command: +If you want to train datasets without nerf-w's train/test split or your own datasets, we provided a light-weight version of the method for general cases. To train with it, you can run the following command ``` -ns-train splatfacto-w --data [PATH] [dataparser] +ns-train splatfacto-w-light --data [PATH] [dataparser] ``` For phototourism, the `dataparser` should be `colmap` and you need to change the colmap path through the CLI because phototourism dataparser does not load 3D points. \ No newline at end of file From 3a46c4274a54e5470ed7b33d85af147ac3105f02 Mon Sep 17 00:00:00 2001 From: Congrong Xu <50019703+KevinXu02@users.noreply.github.com> Date: Thu, 18 Jul 2024 16:03:18 +0800 Subject: [PATCH 2/5] Pass current camera idx in metadata to get output from viser (#3293) allows viser camera to pass clicked training camera idx into metadata --- nerfstudio/viewer/utils.py | 3 +++ nerfstudio/viewer/viewer.py | 19 ++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/nerfstudio/viewer/utils.py b/nerfstudio/viewer/utils.py index 0f5ead912..2fbb5af14 100644 --- a/nerfstudio/viewer/utils.py +++ b/nerfstudio/viewer/utils.py @@ -42,6 +42,8 @@ class CameraState: """Type of camera to render.""" time: float = 0.0 """The rendering time of the camera state.""" + idx: int = 0 + """The index of the current camera.""" def get_camera( @@ -78,6 +80,7 @@ def get_camera( camera_type=camera_state.camera_type, camera_to_worlds=camera_state.c2w.to(torch.float32)[None, ...], times=torch.tensor([camera_state.time], dtype=torch.float32), + metadata={"cam_idx": camera_state.idx}, ) return camera diff --git a/nerfstudio/viewer/viewer.py b/nerfstudio/viewer/viewer.py index bb90ea160..3b01c2e5b 100644 --- a/nerfstudio/viewer/viewer.py +++ b/nerfstudio/viewer/viewer.py @@ -106,6 +106,8 @@ class Viewer: self.train_btn_state: Literal["training", "paused", "completed"] = "training" self._prev_train_state: Literal["training", "paused", "completed"] = "training" self.last_move_time = 0 + # track the camera index that last being clicked + self.current_camera_idx = 0 self.viser_server = viser.ViserServer(host=config.websocket_host, port=websocket_port) # Set the name of the URL either to the share link if available, or the localhost @@ -325,6 +327,7 @@ class Viewer: else CameraType.EQUIRECTANGULAR if camera_type == "Equirectangular" else assert_never(camera_type), + idx=self.current_camera_idx, ) else: camera_state = CameraState( @@ -332,6 +335,7 @@ class Viewer: aspect=client.camera.aspect, c2w=c2w, camera_type=CameraType.PERSPECTIVE, + idx=self.current_camera_idx, ) return camera_state @@ -462,11 +466,16 @@ class Viewer: position=c2w[:3, 3] * VISER_NERFSTUDIO_SCALE_RATIO, ) - @camera_handle.on_click - def _(event: viser.SceneNodePointerEvent[viser.CameraFrustumHandle]) -> None: - with event.client.atomic(): - event.client.camera.position = event.target.position - event.client.camera.wxyz = event.target.wxyz + def create_on_click_callback(capture_idx): + def on_click_callback(event: viser.SceneNodePointerEvent[viser.CameraFrustumHandle]) -> None: + with event.client.atomic(): + event.client.camera.position = event.target.position + event.client.camera.wxyz = event.target.wxyz + self.current_camera_idx = capture_idx + + return on_click_callback + + camera_handle.on_click(create_on_click_callback(idx)) self.camera_handles[idx] = camera_handle self.original_c2w[idx] = c2w From 702886b213a5942b64250b3c4dc632e62237a511 Mon Sep 17 00:00:00 2001 From: "J.Y." <132313008+jb-ye@users.noreply.github.com> Date: Wed, 24 Jul 2024 20:22:59 -0400 Subject: [PATCH 3/5] fix ruff check (#3328) --- nerfstudio/data/pixel_samplers.py | 2 +- nerfstudio/data/utils/nerfstudio_collate.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nerfstudio/data/pixel_samplers.py b/nerfstudio/data/pixel_samplers.py index ad11ee409..080b4016a 100644 --- a/nerfstudio/data/pixel_samplers.py +++ b/nerfstudio/data/pixel_samplers.py @@ -337,7 +337,7 @@ class PixelSampler: collated_batch = { key: value[c, y, x] for key, value in batch.items() - if key != "image_idx" and key != "image" and key != "mask" and key != "depth_image" and value is not None + if key not in ("image_idx", "image", "mask", "depth_image") and value is not None } collated_batch["image"] = torch.cat(all_images, dim=0) diff --git a/nerfstudio/data/utils/nerfstudio_collate.py b/nerfstudio/data/utils/nerfstudio_collate.py index 8c8a633fb..b5b391f54 100644 --- a/nerfstudio/data/utils/nerfstudio_collate.py +++ b/nerfstudio/data/utils/nerfstudio_collate.py @@ -101,7 +101,7 @@ def nerfstudio_collate(batch: Any, extra_mappings: Union[Dict[type, Callable], N storage = elem.storage()._new_shared(numel, device=elem.device) out = elem.new(storage).resize_(len(batch), *list(elem.size())) return torch.stack(batch, 0, out=out) - elif elem_type.__module__ == "numpy" and elem_type.__name__ != "str_" and elem_type.__name__ != "string_": + elif elem_type.__module__ == "numpy" and elem_type.__name__ not in ("str_", "string_"): if elem_type.__name__ in ("ndarray", "memmap"): # array of string classes and object if np_str_obj_array_pattern.search(elem.dtype.str) is not None: From 8c879069710b0fb199fafa1eddbec8efa660ff5f Mon Sep 17 00:00:00 2001 From: Sebastiaan <751205+SharkWipf@users.noreply.github.com> Date: Sat, 27 Jul 2024 21:48:56 +0200 Subject: [PATCH 4/5] Swap out opencv-python for opencv-python-headless (#3336) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 499815b3e..397433cba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ "msgpack_numpy>=0.4.8", "nerfacc==0.5.2", "open3d>=0.16.0", - "opencv-python==4.10.0.84", + "opencv-python-headless==4.10.0.84", "Pillow>=10.3.0", "plotly>=5.7.0", "protobuf<=3.20.3,!=3.20.0", From b0dd3d473094fb0942b357ef436b362df1ef77cd Mon Sep 17 00:00:00 2001 From: Guangyun-Xu <38279883+Guangyun-Xu@users.noreply.github.com> Date: Tue, 30 Jul 2024 03:44:06 +0800 Subject: [PATCH 5/5] make np.array compatible with numpy 2.0 (#3339) * make np.array compatible with numpy 2.0 * Update camera_utils.py --- nerfstudio/cameras/camera_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nerfstudio/cameras/camera_utils.py b/nerfstudio/cameras/camera_utils.py index 185a852a2..3e8785033 100644 --- a/nerfstudio/cameras/camera_utils.py +++ b/nerfstudio/cameras/camera_utils.py @@ -54,7 +54,7 @@ def quaternion_from_matrix(matrix: NDArray, isprecise: bool = False) -> np.ndarr matrix: rotation matrix to obtain quaternion isprecise: if True, input matrix is assumed to be precise rotation matrix and a faster algorithm is used. """ - M = np.array(matrix, dtype=np.float64, copy=False)[:4, :4] + M = np.array(matrix, dtype=np.float64, copy=True)[:4, :4] if isprecise: q = np.empty((4,)) t = np.trace(M)