fixed mc

nerfstudio/process_data/colmap_converter_to_nerfstudio_dataset.py

@@ -101,6 +101,9 @@ class ColmapConverterToNerfstudioDataset(BaseConverterToNerfstudioDataset):
     """If --use-sfm-depth and this flag is True, also export debug images showing Sf overlaid upon input images."""
     same_dimensions: bool = True
     """Whether to assume all images are same dimensions and so to use fast downscaling with no autorotation."""
+    use_single_camera_mode: bool = True
+    """Whether to assume all images taken with the same camera characteristics, set to False for multiple cameras in colmap (only works with hloc sfm_tool).
+    """
 
     @staticmethod
     def default_colmap_path() -> Path:
@@ -136,6 +139,7 @@ class ColmapConverterToNerfstudioDataset(BaseConverterToNerfstudioDataset):
                     image_id_to_depth_path=image_id_to_depth_path,
                     camera_mask_path=camera_mask_path,
                     image_rename_map=image_rename_map,
+                    use_single_camera_mode=self.use_single_camera_mode,
                 )
                 summary_log.append(f"Colmap matched {num_matched_frames} images")
             summary_log.append(colmap_utils.get_matching_summary(num_frames, num_matched_frames))
@@ -194,6 +198,10 @@ class ColmapConverterToNerfstudioDataset(BaseConverterToNerfstudioDataset):
         if self.refine_pixsfm:
             assert sfm_tool == "hloc", "refine_pixsfm only works with sfm_tool hloc"
 
+        # check that sfm_tool is hloc if using use_single_camera_mode
+        if not self.use_single_camera_mode:
+            assert sfm_tool == "hloc", "not_use_single_camera_mode only works with sfm_tool hloc"
+
         # set the image_dir if didn't copy
         if self.skip_image_processing:
             image_dir = self.data
@@ -228,6 +236,7 @@ class ColmapConverterToNerfstudioDataset(BaseConverterToNerfstudioDataset):
                 feature_type=feature_type,
                 matcher_type=matcher_type,
                 refine_pixsfm=self.refine_pixsfm,
+                use_single_camera_mode=self.use_single_camera_mode,
             )
         else:
             raise RuntimeError("Invalid combination of sfm_tool, feature_type, and matcher_type, " "exiting")

nerfstudio/process_data/colmap_utils.py

@@ -395,6 +395,7 @@ def colmap_to_json(
     image_rename_map: Optional[Dict[str, str]] = None,
     ply_filename="sparse_pc.ply",
     keep_original_world_coordinate: bool = False,
+    use_single_camera_mode: bool = True,
 ) -> int:
     """Converts COLMAP's cameras.bin and images.bin to a JSON file.
 
@@ -418,6 +419,13 @@ def colmap_to_json(
     # im_id_to_image = recon.images
     cam_id_to_camera = read_cameras_binary(recon_dir / "cameras.bin")
     im_id_to_image = read_images_binary(recon_dir / "images.bin")
+    if set(cam_id_to_camera.keys()) != {1}:
+        CONSOLE.print(f"[bold yellow]Warning: More than one camera is found in {recon_dir}")
+        print(cam_id_to_camera)
+        use_single_camera_mode = False  # update bool: one camera per frame
+        out = {}  # out = {"camera_model": parse_colmap_camera_params(cam_id_to_camera[1])["camera_model"]}
+    else:  # one camera for all frames
+        out = parse_colmap_camera_params(cam_id_to_camera[1])
 
     frames = []
     for im_id, im_data in im_id_to_image.items():
@@ -455,11 +463,12 @@ def colmap_to_json(
         if image_id_to_depth_path is not None:
             depth_path = image_id_to_depth_path[im_id]
             frame["depth_file_path"] = str(depth_path.relative_to(depth_path.parent.parent))
+
+        if not use_single_camera_mode:  # add the camera parameters for this frame
+            frame.update(parse_colmap_camera_params(cam_id_to_camera[im_data.camera_id]))
+
         frames.append(frame)
 
-    if set(cam_id_to_camera.keys()) != {1}:
-        raise RuntimeError("Only single camera shared for all images is supported.")
-    out = parse_colmap_camera_params(cam_id_to_camera[1])
     out["frames"] = frames
 
     applied_transform = None

nerfstudio/process_data/hloc_utils.py

@@ -71,6 +71,7 @@ def run_hloc(
     ] = "superglue",
     num_matched: int = 50,
     refine_pixsfm: bool = False,
+    use_single_camera_mode: bool = True,
 ) -> None:
     """Runs hloc on the images.
 
@@ -85,6 +86,7 @@ def run_hloc(
         matcher_type: Type of feature matcher to use.
         num_matched: Number of image pairs for loc.
         refine_pixsfm: If True, refine the reconstruction using pixel-perfect-sfm.
+        use_single_camera_mode: If True, uses one camera for all frames. Otherwise uses one camera per frame.
     """
     if not _HAS_HLOC:
         CONSOLE.print(
@@ -119,6 +121,12 @@ def run_hloc(
     match_features.main(matcher_conf, sfm_pairs, features=features, matches=matches)  # type: ignore
 
     image_options = pycolmap.ImageReaderOptions(camera_model=camera_model.value)  # type: ignore
+
+    if use_single_camera_mode:  # one camera per all frames
+        camera_mode = pycolmap.CameraMode.SINGLE  # type: ignore
+    else:  # one camera per frame
+        camera_mode = pycolmap.CameraMode.PER_IMAGE  # type: ignore
+
     if refine_pixsfm:
         sfm = PixSfM(  # type: ignore
             conf={
@@ -134,7 +142,7 @@ def run_hloc(
             features,
             matches,
             image_list=references,
-            camera_mode=pycolmap.CameraMode.SINGLE,  # type: ignore
+            camera_mode=camera_mode,  # type: ignore
             image_options=image_options,
             verbose=verbose,
         )
@@ -147,7 +155,7 @@ def run_hloc(
             sfm_pairs,
             features,
             matches,
-            camera_mode=pycolmap.CameraMode.SINGLE,  # type: ignore
+            camera_mode=camera_mode,  # type: ignore
             image_options=image_options,
             verbose=verbose,
         )

nerfstudio/scripts/exporter.py

@@ -491,6 +491,20 @@ class ExportGaussianSplat(Exporter):
     Export 3D Gaussian Splatting model to a .ply
     """
 
+    use_bounding_box: Optional[bool] = False
+    """Only query points within the bounding box"""
+    bounding_box_min: Optional[Tuple[float, float, float]] = (-1, -1, -1)
+    """Minimum of the bounding box, used if use_bounding_box is True."""
+    bounding_box_max: Optional[Tuple[float, float, float]] = (1, 1, 1)
+    """Maximum of the bounding box, used if use_bounding_box is True."""
+
+    obb_center: Optional[Tuple[float, float, float]] = None
+    """Center of the oriented bounding box."""
+    obb_rotation: Optional[Tuple[float, float, float]] = None
+    """Rotation of the oriented bounding box. Expressed as RPY Euler angles in radians"""
+    obb_scale: Optional[Tuple[float, float, float]] = None
+    """Scale of the oriented bounding box along each axis."""
+
     @staticmethod
     def write_ply(filename: str, count: int, map_to_tensors: OrderedDict[str, np.ndarray]):
         """
@@ -634,6 +648,37 @@ class ExportGaussianSplat(Exporter):
 
             map_to_tensors["veggie"] = min_inds.float().cpu().numpy()
 
+            if (
+                self.use_bounding_box
+                and self.obb_center is not None
+                and self.obb_rotation is not None
+                and self.obb_scale is not None
+            ):
+                crop_obb = OrientedBox.from_params(self.obb_center, self.obb_rotation, self.obb_scale)
+                assert crop_obb is not None
+                mask = crop_obb.within(torch.from_numpy(positions)).numpy()
+                for k, t in map_to_tensors.items():
+                    map_to_tensors[k] = map_to_tensors[k][mask]
+
+                n = map_to_tensors["x"].shape[0]
+                count = n
+            elif self.use_bounding_box:
+                assert self.bounding_box_min is not None and self.bounding_box_max is not None
+                x_mask = np.logical_and(
+                    positions[:, 0] >= self.bounding_box_min[0], positions[:, 0] <= self.bounding_box_max[0]
+                )
+                y_mask = np.logical_and(
+                    positions[:, 1] >= self.bounding_box_min[1], positions[:, 1] <= self.bounding_box_max[1]
+                )
+                z_mask = np.logical_and(
+                    positions[:, 2] >= self.bounding_box_min[2], positions[:, 2] <= self.bounding_box_max[2]
+                )
+                mask = np.logical_and(np.logical_and(x_mask, y_mask), z_mask)
+                for k, t in map_to_tensors.items():
+                    map_to_tensors[k] = map_to_tensors[k][mask]
+
+                n = map_to_tensors["x"].shape[0]
+                count = n
         # post optimization, it is possible have NaN/Inf values in some attributes
         # to ensure the exported ply file has finite values, we enforce finite filters.
         select = np.ones(n, dtype=bool)

nerfstudio/viewer/export_panel.py

@@ -204,6 +204,8 @@ def populate_splat_tab(
                     "ns-export gaussian-splat",
                     f"--load-config {config_path}",
                     f"--output-dir {output_directory.value}",
+                    f"--use_bounding_box {control_panel.crop_viewport}",
+                    get_crop_string(control_panel.crop_obb),
                 ]
             )
             show_command_modal(event.client, "splat", command)