From db93476b6a3d7ae0114f3c45f0ce70a3a15853be Mon Sep 17 00:00:00 2001 From: dberga Date: Wed, 24 Apr 2024 18:43:45 +0200 Subject: [PATCH 1/3] Hloc fix for image_ids (camera per image) (#3081) * changed run_hloc args to fill image_ids (added image_list and changed camera_mode to PER_IMAGE) * added parameter same_camera on ns-process-data for multifocal cases in hloc. Also updated colmap_to_json to detect if there is one or several cameras per frame while keeping the camera type specifications * ruff-formatted changes in colmap_utils.py * use_single_camera_mode update args, including colmap_utils and hloc_utils * changed multicamera warning's print to CONSOLE.print * fix path typo console print * rewritten use-single-camera-mode comment in ns-process-data * (fix) inverted bool for colmap_utils * change im_id to im_data.camera_id --------- Co-authored-by: Jianbo Ye --- .../colmap_converter_to_nerfstudio_dataset.py | 9 +++++++++ nerfstudio/process_data/colmap_utils.py | 15 ++++++++++++--- nerfstudio/process_data/hloc_utils.py | 12 ++++++++++-- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/nerfstudio/process_data/colmap_converter_to_nerfstudio_dataset.py b/nerfstudio/process_data/colmap_converter_to_nerfstudio_dataset.py index 35d45b905..b17243329 100644 --- a/nerfstudio/process_data/colmap_converter_to_nerfstudio_dataset.py +++ b/nerfstudio/process_data/colmap_converter_to_nerfstudio_dataset.py @@ -101,6 +101,9 @@ class ColmapConverterToNerfstudioDataset(BaseConverterToNerfstudioDataset): """If --use-sfm-depth and this flag is True, also export debug images showing Sf overlaid upon input images.""" same_dimensions: bool = True """Whether to assume all images are same dimensions and so to use fast downscaling with no autorotation.""" + use_single_camera_mode: bool = True + """Whether to assume all images taken with the same camera characteristics, set to False for multiple cameras in colmap (only works with hloc sfm_tool). + """ @staticmethod def default_colmap_path() -> Path: @@ -136,6 +139,7 @@ class ColmapConverterToNerfstudioDataset(BaseConverterToNerfstudioDataset): image_id_to_depth_path=image_id_to_depth_path, camera_mask_path=camera_mask_path, image_rename_map=image_rename_map, + use_single_camera_mode=self.use_single_camera_mode, ) summary_log.append(f"Colmap matched {num_matched_frames} images") summary_log.append(colmap_utils.get_matching_summary(num_frames, num_matched_frames)) @@ -194,6 +198,10 @@ class ColmapConverterToNerfstudioDataset(BaseConverterToNerfstudioDataset): if self.refine_pixsfm: assert sfm_tool == "hloc", "refine_pixsfm only works with sfm_tool hloc" + # check that sfm_tool is hloc if using use_single_camera_mode + if not self.use_single_camera_mode: + assert sfm_tool == "hloc", "not_use_single_camera_mode only works with sfm_tool hloc" + # set the image_dir if didn't copy if self.skip_image_processing: image_dir = self.data @@ -228,6 +236,7 @@ class ColmapConverterToNerfstudioDataset(BaseConverterToNerfstudioDataset): feature_type=feature_type, matcher_type=matcher_type, refine_pixsfm=self.refine_pixsfm, + use_single_camera_mode=self.use_single_camera_mode, ) else: raise RuntimeError("Invalid combination of sfm_tool, feature_type, and matcher_type, " "exiting") diff --git a/nerfstudio/process_data/colmap_utils.py b/nerfstudio/process_data/colmap_utils.py index 7860d1050..1d9405c81 100644 --- a/nerfstudio/process_data/colmap_utils.py +++ b/nerfstudio/process_data/colmap_utils.py @@ -395,6 +395,7 @@ def colmap_to_json( image_rename_map: Optional[Dict[str, str]] = None, ply_filename="sparse_pc.ply", keep_original_world_coordinate: bool = False, + use_single_camera_mode: bool = True, ) -> int: """Converts COLMAP's cameras.bin and images.bin to a JSON file. @@ -418,6 +419,13 @@ def colmap_to_json( # im_id_to_image = recon.images cam_id_to_camera = read_cameras_binary(recon_dir / "cameras.bin") im_id_to_image = read_images_binary(recon_dir / "images.bin") + if set(cam_id_to_camera.keys()) != {1}: + CONSOLE.print(f"[bold yellow]Warning: More than one camera is found in {recon_dir}") + print(cam_id_to_camera) + use_single_camera_mode = False # update bool: one camera per frame + out = {} # out = {"camera_model": parse_colmap_camera_params(cam_id_to_camera[1])["camera_model"]} + else: # one camera for all frames + out = parse_colmap_camera_params(cam_id_to_camera[1]) frames = [] for im_id, im_data in im_id_to_image.items(): @@ -455,11 +463,12 @@ def colmap_to_json( if image_id_to_depth_path is not None: depth_path = image_id_to_depth_path[im_id] frame["depth_file_path"] = str(depth_path.relative_to(depth_path.parent.parent)) + + if not use_single_camera_mode: # add the camera parameters for this frame + frame.update(parse_colmap_camera_params(cam_id_to_camera[im_data.camera_id])) + frames.append(frame) - if set(cam_id_to_camera.keys()) != {1}: - raise RuntimeError("Only single camera shared for all images is supported.") - out = parse_colmap_camera_params(cam_id_to_camera[1]) out["frames"] = frames applied_transform = None diff --git a/nerfstudio/process_data/hloc_utils.py b/nerfstudio/process_data/hloc_utils.py index b7167efd8..eda9ee5b0 100644 --- a/nerfstudio/process_data/hloc_utils.py +++ b/nerfstudio/process_data/hloc_utils.py @@ -71,6 +71,7 @@ def run_hloc( ] = "superglue", num_matched: int = 50, refine_pixsfm: bool = False, + use_single_camera_mode: bool = True, ) -> None: """Runs hloc on the images. @@ -85,6 +86,7 @@ def run_hloc( matcher_type: Type of feature matcher to use. num_matched: Number of image pairs for loc. refine_pixsfm: If True, refine the reconstruction using pixel-perfect-sfm. + use_single_camera_mode: If True, uses one camera for all frames. Otherwise uses one camera per frame. """ if not _HAS_HLOC: CONSOLE.print( @@ -119,6 +121,12 @@ def run_hloc( match_features.main(matcher_conf, sfm_pairs, features=features, matches=matches) # type: ignore image_options = pycolmap.ImageReaderOptions(camera_model=camera_model.value) # type: ignore + + if use_single_camera_mode: # one camera per all frames + camera_mode = pycolmap.CameraMode.SINGLE # type: ignore + else: # one camera per frame + camera_mode = pycolmap.CameraMode.PER_IMAGE # type: ignore + if refine_pixsfm: sfm = PixSfM( # type: ignore conf={ @@ -134,7 +142,7 @@ def run_hloc( features, matches, image_list=references, - camera_mode=pycolmap.CameraMode.SINGLE, # type: ignore + camera_mode=camera_mode, # type: ignore image_options=image_options, verbose=verbose, ) @@ -147,7 +155,7 @@ def run_hloc( sfm_pairs, features, matches, - camera_mode=pycolmap.CameraMode.SINGLE, # type: ignore + camera_mode=camera_mode, # type: ignore image_options=image_options, verbose=verbose, ) From 35d970a0d22508e88f74f8b24d2230a8d99fc8f5 Mon Sep 17 00:00:00 2001 From: Rohan Mathur Date: Wed, 24 Apr 2024 15:00:58 -0700 Subject: [PATCH 2/3] added bounding box to splat export params --- nerfstudio/scripts/exporter.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/nerfstudio/scripts/exporter.py b/nerfstudio/scripts/exporter.py index 29c7ebc22..70aed8eef 100644 --- a/nerfstudio/scripts/exporter.py +++ b/nerfstudio/scripts/exporter.py @@ -57,6 +57,8 @@ class Exporter: """Path to the config YAML file.""" output_dir: Path """Path to the output directory.""" + bbox: Optional[Tuple] = None + """Bounding box for the scene in the form of ((x_min, x_max), (y_min, y_max), (z_min, z_max)).""" def validate_pipeline(normal_method: str, normal_output_name: str, pipeline: Pipeline) -> None: @@ -585,6 +587,17 @@ class ExportGaussianSplat(Exporter): for i in range(4): map_to_tensors[f"rot_{i}"] = quats[:, i, None] + if self.bbox: + bbox = np.array(self.bbox, dtype=np.float32).reshape(3, 2) + x_mask = np.logical_and(positions[:, 0] >= bbox[0, 0], positions[:, 0] <= bbox[0, 1]) + y_mask = np.logical_and(positions[:, 1] >= bbox[1, 0], positions[:, 1] <= bbox[1, 1]) + z_mask = np.logical_and(positions[:, 2] >= bbox[2, 0], positions[:, 2] <= bbox[2, 1]) + mask = np.logical_and(np.logical_and(x_mask, y_mask), z_mask) + for k, t in map_to_tensors.items(): + map_to_tensors[k] = map_to_tensors[k][mask] + + n = map_to_tensors["x"].shape[0] + count = n # post optimization, it is possible have NaN/Inf values in some attributes # to ensure the exported ply file has finite values, we enforce finite filters. select = np.ones(n, dtype=bool) From 43188cfaf470563bd2025c965542291e212f2b0a Mon Sep 17 00:00:00 2001 From: Rohan Mathur Date: Wed, 24 Apr 2024 17:28:01 -0700 Subject: [PATCH 3/3] finished bounding box for splat --- nerfstudio/scripts/exporter.py | 46 ++++++++++++++++++++++++++----- nerfstudio/viewer/export_panel.py | 2 ++ 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/nerfstudio/scripts/exporter.py b/nerfstudio/scripts/exporter.py index 70aed8eef..51fb29dbe 100644 --- a/nerfstudio/scripts/exporter.py +++ b/nerfstudio/scripts/exporter.py @@ -57,8 +57,6 @@ class Exporter: """Path to the config YAML file.""" output_dir: Path """Path to the output directory.""" - bbox: Optional[Tuple] = None - """Bounding box for the scene in the form of ((x_min, x_max), (y_min, y_max), (z_min, z_max)).""" def validate_pipeline(normal_method: str, normal_output_name: str, pipeline: Pipeline) -> None: @@ -493,6 +491,20 @@ class ExportGaussianSplat(Exporter): Export 3D Gaussian Splatting model to a .ply """ + use_bounding_box: Optional[bool] = False + """Only query points within the bounding box""" + bounding_box_min: Optional[Tuple[float, float, float]] = (-1, -1, -1) + """Minimum of the bounding box, used if use_bounding_box is True.""" + bounding_box_max: Optional[Tuple[float, float, float]] = (1, 1, 1) + """Maximum of the bounding box, used if use_bounding_box is True.""" + + obb_center: Optional[Tuple[float, float, float]] = None + """Center of the oriented bounding box.""" + obb_rotation: Optional[Tuple[float, float, float]] = None + """Rotation of the oriented bounding box. Expressed as RPY Euler angles in radians""" + obb_scale: Optional[Tuple[float, float, float]] = None + """Scale of the oriented bounding box along each axis.""" + @staticmethod def write_ply(filename: str, count: int, map_to_tensors: OrderedDict[str, np.ndarray]): """ @@ -587,11 +599,31 @@ class ExportGaussianSplat(Exporter): for i in range(4): map_to_tensors[f"rot_{i}"] = quats[:, i, None] - if self.bbox: - bbox = np.array(self.bbox, dtype=np.float32).reshape(3, 2) - x_mask = np.logical_and(positions[:, 0] >= bbox[0, 0], positions[:, 0] <= bbox[0, 1]) - y_mask = np.logical_and(positions[:, 1] >= bbox[1, 0], positions[:, 1] <= bbox[1, 1]) - z_mask = np.logical_and(positions[:, 2] >= bbox[2, 0], positions[:, 2] <= bbox[2, 1]) + if ( + self.use_bounding_box + and self.obb_center is not None + and self.obb_rotation is not None + and self.obb_scale is not None + ): + crop_obb = OrientedBox.from_params(self.obb_center, self.obb_rotation, self.obb_scale) + assert crop_obb is not None + mask = crop_obb.within(torch.from_numpy(positions)).numpy() + for k, t in map_to_tensors.items(): + map_to_tensors[k] = map_to_tensors[k][mask] + + n = map_to_tensors["x"].shape[0] + count = n + elif self.use_bounding_box: + assert self.bounding_box_min is not None and self.bounding_box_max is not None + x_mask = np.logical_and( + positions[:, 0] >= self.bounding_box_min[0], positions[:, 0] <= self.bounding_box_max[0] + ) + y_mask = np.logical_and( + positions[:, 1] >= self.bounding_box_min[1], positions[:, 1] <= self.bounding_box_max[1] + ) + z_mask = np.logical_and( + positions[:, 2] >= self.bounding_box_min[2], positions[:, 2] <= self.bounding_box_max[2] + ) mask = np.logical_and(np.logical_and(x_mask, y_mask), z_mask) for k, t in map_to_tensors.items(): map_to_tensors[k] = map_to_tensors[k][mask] diff --git a/nerfstudio/viewer/export_panel.py b/nerfstudio/viewer/export_panel.py index 76808a272..a8c03efb8 100644 --- a/nerfstudio/viewer/export_panel.py +++ b/nerfstudio/viewer/export_panel.py @@ -204,6 +204,8 @@ def populate_splat_tab( "ns-export gaussian-splat", f"--load-config {config_path}", f"--output-dir {output_directory.value}", + f"--use_bounding_box {control_panel.crop_viewport}", + get_crop_string(control_panel.crop_obb), ] ) show_command_modal(event.client, "splat", command)