From 9a0ecb6fd7a4518552504d3d32ae8ae80d38ed21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Devernay?= <124616582+f-dy@users.noreply.github.com> Date: Mon, 1 May 2023 14:44:00 -0700 Subject: [PATCH] Support equirectangular sensor with metashape (#1841) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Document how to use equirectangular sensor with metashape * equirectangular sensor is actually supported by nerfstudio! * Update custom_dataset.md * equirectangular: fix COLMAP workflow * metashape_utils.py: black --------- Co-authored-by: Frédéric Devernay --- docs/quickstart/custom_dataset.md | 2 ++ .../images_to_nerstudio_dataset.py | 1 + nerfstudio/process_data/metashape_utils.py | 36 ++++++++++++------- nerfstudio/process_data/process_data_utils.py | 3 +- .../video_to_nerfstudio_dataset.py | 2 ++ 5 files changed, 31 insertions(+), 13 deletions(-) diff --git a/docs/quickstart/custom_dataset.md b/docs/quickstart/custom_dataset.md index a83d59218..ebd77a5f5 100644 --- a/docs/quickstart/custom_dataset.md +++ b/docs/quickstart/custom_dataset.md @@ -259,6 +259,8 @@ ns-train nerfacto --data {output directory} ## Metashape +All images must use the same sensor type (but multiple sensors are supported). + 1. Align your images using Metashape. `File -> Workflow -> Align Photos...` ```{image} https://user-images.githubusercontent.com/3310961/203389662-12760210-2b52-49d4-ab21-4f23bfa4a2b3.png diff --git a/nerfstudio/process_data/images_to_nerstudio_dataset.py b/nerfstudio/process_data/images_to_nerstudio_dataset.py index 477c94c20..dc6163816 100644 --- a/nerfstudio/process_data/images_to_nerstudio_dataset.py +++ b/nerfstudio/process_data/images_to_nerstudio_dataset.py @@ -56,6 +56,7 @@ class ImagesToNerfstudioDataset(ColmapConverterToNerfstudioDataset): self.data = equirect_utils.generate_planar_projections_from_equirectangular( self.data, pers_size, self.images_per_equirect, crop_factor=self.crop_factor ) + self.camera_type = "perspective" summary_log = [] diff --git a/nerfstudio/process_data/metashape_utils.py b/nerfstudio/process_data/metashape_utils.py index 0e4a7593e..4e0b1a680 100644 --- a/nerfstudio/process_data/metashape_utils.py +++ b/nerfstudio/process_data/metashape_utils.py @@ -61,12 +61,17 @@ def metashape_to_json( # pylint: disable=too-many-statements if sensors is None: raise ValueError("No sensors found") - calibrated_sensors = [sensor for sensor in sensors if sensor.find("calibration")] + calibrated_sensors = [ + sensor for sensor in sensors if sensor.get("type") == "spherical" or sensor.find("calibration") + ] if not calibrated_sensors: raise ValueError("No calibrated sensor found in Metashape XML") sensor_type = [s.get("type") for s in calibrated_sensors] if sensor_type.count(sensor_type[0]) != len(sensor_type): - raise ValueError("All Metashape sensors do not have the same sensor type") + raise ValueError( + "All Metashape sensors do not have the same sensor type. " + "nerfstudio does not support per-frame camera_model." + ) data = {} if sensor_type[0] == "frame": data["camera_model"] = CAMERA_MODELS["perspective"].value @@ -87,18 +92,25 @@ def metashape_to_json( # pylint: disable=too-many-statements s["h"] = int(resolution.get("height")) # type: ignore calib = sensor.find("calibration") - f = calib.find("f") - if f is not None: + if calib is None: + assert sensor_type[0] == "spherical", "Only spherical sensors should have no intrinsics" + s["fl_x"] = s["w"] / 2.0 + s["fl_y"] = s["h"] + s["cx"] = s["w"] / 2.0 + s["cy"] = s["h"] / 2.0 + else: + f = calib.find("f") + assert f is not None, "Focal length not found in Metashape xml" s["fl_x"] = s["fl_y"] = float(f.text) # type: ignore - s["cx"] = _find_param(calib, "cx") + s["w"] / 2.0 # type: ignore - s["cy"] = _find_param(calib, "cy") + s["h"] / 2.0 # type: ignore + s["cx"] = _find_param(calib, "cx") + s["w"] / 2.0 # type: ignore + s["cy"] = _find_param(calib, "cy") + s["h"] / 2.0 # type: ignore - s["k1"] = _find_param(calib, "k1") - s["k2"] = _find_param(calib, "k2") - s["k3"] = _find_param(calib, "k3") - s["k4"] = _find_param(calib, "k4") - s["p1"] = _find_param(calib, "p1") - s["p2"] = _find_param(calib, "p2") + s["k1"] = _find_param(calib, "k1") + s["k2"] = _find_param(calib, "k2") + s["k3"] = _find_param(calib, "k3") + s["k4"] = _find_param(calib, "k4") + s["p1"] = _find_param(calib, "p1") + s["p2"] = _find_param(calib, "p2") sensor_dict[sensor.get("id")] = s diff --git a/nerfstudio/process_data/process_data_utils.py b/nerfstudio/process_data/process_data_utils.py index a977df8b2..12d440772 100644 --- a/nerfstudio/process_data/process_data_utils.py +++ b/nerfstudio/process_data/process_data_utils.py @@ -39,12 +39,13 @@ class CameraModel(Enum): OPENCV = "OPENCV" OPENCV_FISHEYE = "OPENCV_FISHEYE" + EQUIRECTANGULAR = "EQUIRECTANGULAR" CAMERA_MODELS = { "perspective": CameraModel.OPENCV, "fisheye": CameraModel.OPENCV_FISHEYE, - "equirectangular": CameraModel.OPENCV, + "equirectangular": CameraModel.EQUIRECTANGULAR, } diff --git a/nerfstudio/process_data/video_to_nerfstudio_dataset.py b/nerfstudio/process_data/video_to_nerfstudio_dataset.py index 405231fcf..f8054ff42 100644 --- a/nerfstudio/process_data/video_to_nerfstudio_dataset.py +++ b/nerfstudio/process_data/video_to_nerfstudio_dataset.py @@ -91,6 +91,8 @@ class VideoToNerfstudioDataset(ColmapConverterToNerfstudioDataset): # remove the temp_images folder shutil.rmtree(self.output_dir / "temp_images", ignore_errors=True) + self.camera_type = "perspective" + # # Create mask mask_path = process_data_utils.save_mask( image_dir=self.image_dir,