From db93476b6a3d7ae0114f3c45f0ce70a3a15853be Mon Sep 17 00:00:00 2001 From: dberga Date: Wed, 24 Apr 2024 18:43:45 +0200 Subject: [PATCH] Hloc fix for image_ids (camera per image) (#3081) * changed run_hloc args to fill image_ids (added image_list and changed camera_mode to PER_IMAGE) * added parameter same_camera on ns-process-data for multifocal cases in hloc. Also updated colmap_to_json to detect if there is one or several cameras per frame while keeping the camera type specifications * ruff-formatted changes in colmap_utils.py * use_single_camera_mode update args, including colmap_utils and hloc_utils * changed multicamera warning's print to CONSOLE.print * fix path typo console print * rewritten use-single-camera-mode comment in ns-process-data * (fix) inverted bool for colmap_utils * change im_id to im_data.camera_id --------- Co-authored-by: Jianbo Ye --- .../colmap_converter_to_nerfstudio_dataset.py | 9 +++++++++ nerfstudio/process_data/colmap_utils.py | 15 ++++++++++++--- nerfstudio/process_data/hloc_utils.py | 12 ++++++++++-- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/nerfstudio/process_data/colmap_converter_to_nerfstudio_dataset.py b/nerfstudio/process_data/colmap_converter_to_nerfstudio_dataset.py index 35d45b905..b17243329 100644 --- a/nerfstudio/process_data/colmap_converter_to_nerfstudio_dataset.py +++ b/nerfstudio/process_data/colmap_converter_to_nerfstudio_dataset.py @@ -101,6 +101,9 @@ class ColmapConverterToNerfstudioDataset(BaseConverterToNerfstudioDataset): """If --use-sfm-depth and this flag is True, also export debug images showing Sf overlaid upon input images.""" same_dimensions: bool = True """Whether to assume all images are same dimensions and so to use fast downscaling with no autorotation.""" + use_single_camera_mode: bool = True + """Whether to assume all images taken with the same camera characteristics, set to False for multiple cameras in colmap (only works with hloc sfm_tool). + """ @staticmethod def default_colmap_path() -> Path: @@ -136,6 +139,7 @@ class ColmapConverterToNerfstudioDataset(BaseConverterToNerfstudioDataset): image_id_to_depth_path=image_id_to_depth_path, camera_mask_path=camera_mask_path, image_rename_map=image_rename_map, + use_single_camera_mode=self.use_single_camera_mode, ) summary_log.append(f"Colmap matched {num_matched_frames} images") summary_log.append(colmap_utils.get_matching_summary(num_frames, num_matched_frames)) @@ -194,6 +198,10 @@ class ColmapConverterToNerfstudioDataset(BaseConverterToNerfstudioDataset): if self.refine_pixsfm: assert sfm_tool == "hloc", "refine_pixsfm only works with sfm_tool hloc" + # check that sfm_tool is hloc if using use_single_camera_mode + if not self.use_single_camera_mode: + assert sfm_tool == "hloc", "not_use_single_camera_mode only works with sfm_tool hloc" + # set the image_dir if didn't copy if self.skip_image_processing: image_dir = self.data @@ -228,6 +236,7 @@ class ColmapConverterToNerfstudioDataset(BaseConverterToNerfstudioDataset): feature_type=feature_type, matcher_type=matcher_type, refine_pixsfm=self.refine_pixsfm, + use_single_camera_mode=self.use_single_camera_mode, ) else: raise RuntimeError("Invalid combination of sfm_tool, feature_type, and matcher_type, " "exiting") diff --git a/nerfstudio/process_data/colmap_utils.py b/nerfstudio/process_data/colmap_utils.py index 7860d1050..1d9405c81 100644 --- a/nerfstudio/process_data/colmap_utils.py +++ b/nerfstudio/process_data/colmap_utils.py @@ -395,6 +395,7 @@ def colmap_to_json( image_rename_map: Optional[Dict[str, str]] = None, ply_filename="sparse_pc.ply", keep_original_world_coordinate: bool = False, + use_single_camera_mode: bool = True, ) -> int: """Converts COLMAP's cameras.bin and images.bin to a JSON file. @@ -418,6 +419,13 @@ def colmap_to_json( # im_id_to_image = recon.images cam_id_to_camera = read_cameras_binary(recon_dir / "cameras.bin") im_id_to_image = read_images_binary(recon_dir / "images.bin") + if set(cam_id_to_camera.keys()) != {1}: + CONSOLE.print(f"[bold yellow]Warning: More than one camera is found in {recon_dir}") + print(cam_id_to_camera) + use_single_camera_mode = False # update bool: one camera per frame + out = {} # out = {"camera_model": parse_colmap_camera_params(cam_id_to_camera[1])["camera_model"]} + else: # one camera for all frames + out = parse_colmap_camera_params(cam_id_to_camera[1]) frames = [] for im_id, im_data in im_id_to_image.items(): @@ -455,11 +463,12 @@ def colmap_to_json( if image_id_to_depth_path is not None: depth_path = image_id_to_depth_path[im_id] frame["depth_file_path"] = str(depth_path.relative_to(depth_path.parent.parent)) + + if not use_single_camera_mode: # add the camera parameters for this frame + frame.update(parse_colmap_camera_params(cam_id_to_camera[im_data.camera_id])) + frames.append(frame) - if set(cam_id_to_camera.keys()) != {1}: - raise RuntimeError("Only single camera shared for all images is supported.") - out = parse_colmap_camera_params(cam_id_to_camera[1]) out["frames"] = frames applied_transform = None diff --git a/nerfstudio/process_data/hloc_utils.py b/nerfstudio/process_data/hloc_utils.py index b7167efd8..eda9ee5b0 100644 --- a/nerfstudio/process_data/hloc_utils.py +++ b/nerfstudio/process_data/hloc_utils.py @@ -71,6 +71,7 @@ def run_hloc( ] = "superglue", num_matched: int = 50, refine_pixsfm: bool = False, + use_single_camera_mode: bool = True, ) -> None: """Runs hloc on the images. @@ -85,6 +86,7 @@ def run_hloc( matcher_type: Type of feature matcher to use. num_matched: Number of image pairs for loc. refine_pixsfm: If True, refine the reconstruction using pixel-perfect-sfm. + use_single_camera_mode: If True, uses one camera for all frames. Otherwise uses one camera per frame. """ if not _HAS_HLOC: CONSOLE.print( @@ -119,6 +121,12 @@ def run_hloc( match_features.main(matcher_conf, sfm_pairs, features=features, matches=matches) # type: ignore image_options = pycolmap.ImageReaderOptions(camera_model=camera_model.value) # type: ignore + + if use_single_camera_mode: # one camera per all frames + camera_mode = pycolmap.CameraMode.SINGLE # type: ignore + else: # one camera per frame + camera_mode = pycolmap.CameraMode.PER_IMAGE # type: ignore + if refine_pixsfm: sfm = PixSfM( # type: ignore conf={ @@ -134,7 +142,7 @@ def run_hloc( features, matches, image_list=references, - camera_mode=pycolmap.CameraMode.SINGLE, # type: ignore + camera_mode=camera_mode, # type: ignore image_options=image_options, verbose=verbose, ) @@ -147,7 +155,7 @@ def run_hloc( sfm_pairs, features, matches, - camera_mode=pycolmap.CameraMode.SINGLE, # type: ignore + camera_mode=camera_mode, # type: ignore image_options=image_options, verbose=verbose, )