From aa6c0526602f6ca4c5f41449163ee5d6ca37f4c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=C3=A1=C5=A1=20Kulh=C3=A1nek?= Date: Sun, 25 Jun 2023 19:08:17 +0200 Subject: [PATCH] Jkulhanek/colmap dataparser (#2115) * Add COLMAP dataparser implementation * Fix downscale_images input * Add help to COLMAP dataparser --- nerfstudio/configs/dataparser_configs.py | 14 +- .../data/dataparsers/colmap_dataparser.py | 403 ++++++++++++++++++ 2 files changed, 408 insertions(+), 9 deletions(-) create mode 100644 nerfstudio/data/dataparsers/colmap_dataparser.py diff --git a/nerfstudio/configs/dataparser_configs.py b/nerfstudio/configs/dataparser_configs.py index 80b915151..46a728141 100644 --- a/nerfstudio/configs/dataparser_configs.py +++ b/nerfstudio/configs/dataparser_configs.py @@ -20,23 +20,18 @@ from typing import TYPE_CHECKING import tyro -from nerfstudio.data.dataparsers.arkitscenes_dataparser import ( - ARKitScenesDataParserConfig, -) +from nerfstudio.data.dataparsers.arkitscenes_dataparser import ARKitScenesDataParserConfig from nerfstudio.data.dataparsers.base_dataparser import DataParserConfig from nerfstudio.data.dataparsers.blender_dataparser import BlenderDataParserConfig +from nerfstudio.data.dataparsers.colmap_dataparser import ColmapDataParserConfig from nerfstudio.data.dataparsers.dnerf_dataparser import DNeRFDataParserConfig from nerfstudio.data.dataparsers.dycheck_dataparser import DycheckDataParserConfig -from nerfstudio.data.dataparsers.instant_ngp_dataparser import ( - InstantNGPDataParserConfig, -) +from nerfstudio.data.dataparsers.instant_ngp_dataparser import InstantNGPDataParserConfig from nerfstudio.data.dataparsers.minimal_dataparser import MinimalDataParserConfig from nerfstudio.data.dataparsers.nerfosr_dataparser import NeRFOSRDataParserConfig from nerfstudio.data.dataparsers.nerfstudio_dataparser import NerfstudioDataParserConfig from nerfstudio.data.dataparsers.nuscenes_dataparser import NuScenesDataParserConfig -from nerfstudio.data.dataparsers.phototourism_dataparser import ( - PhototourismDataParserConfig, -) +from nerfstudio.data.dataparsers.phototourism_dataparser import PhototourismDataParserConfig from nerfstudio.data.dataparsers.scannet_dataparser import ScanNetDataParserConfig from nerfstudio.data.dataparsers.sdfstudio_dataparser import SDFStudioDataParserConfig from nerfstudio.data.dataparsers.sitcoms3d_dataparser import Sitcoms3DDataParserConfig @@ -56,6 +51,7 @@ dataparsers = { "sdfstudio-data": SDFStudioDataParserConfig(), "nerfosr-data": NeRFOSRDataParserConfig(), "sitcoms3d-data": Sitcoms3DDataParserConfig(), + "colmap": ColmapDataParserConfig(), } external_dataparsers = discover_dataparsers() diff --git a/nerfstudio/data/dataparsers/colmap_dataparser.py b/nerfstudio/data/dataparsers/colmap_dataparser.py new file mode 100644 index 000000000..bb3284f33 --- /dev/null +++ b/nerfstudio/data/dataparsers/colmap_dataparser.py @@ -0,0 +1,403 @@ +# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Data parser for nerfstudio datasets. """ + +from __future__ import annotations + +import math +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import List, Literal, Optional, Type + +import numpy as np +import torch +from PIL import Image +from rich.prompt import Confirm + +from nerfstudio.cameras import camera_utils +from nerfstudio.cameras.cameras import CAMERA_MODEL_TO_TYPE, Cameras +from nerfstudio.data.dataparsers.base_dataparser import DataParser, DataParserConfig, DataparserOutputs +from nerfstudio.data.scene_box import SceneBox +from nerfstudio.data.utils import colmap_parsing_utils as colmap_utils +from nerfstudio.process_data.colmap_utils import parse_colmap_camera_params +from nerfstudio.process_data.process_data_utils import downscale_images +from nerfstudio.utils.rich_utils import CONSOLE + +MAX_AUTO_RESOLUTION = 1600 + + +@dataclass +class ColmapDataParserConfig(DataParserConfig): + """Nerfstudio dataset config""" + + _target: Type = field(default_factory=lambda: ColmapDataParser) + """target class to instantiate""" + data: Path = Path() + """Directory or explicit json file path specifying location of data.""" + scale_factor: float = 1.0 + """How much to scale the camera origins by.""" + downscale_factor: Optional[int] = None + """How much to downscale images. If not set, images are chosen such that the max dimension is <1600px.""" + scene_scale: float = 1.0 + """How much to scale the region of interest by.""" + orientation_method: Literal["pca", "up", "vertical", "none"] = "up" + """The method to use for orientation.""" + center_method: Literal["poses", "focus", "none"] = "poses" + """The method to use to center the poses.""" + auto_scale_poses: bool = True + """Whether to automatically scale the poses to fit in +/- 1 bounding box.""" + train_split_fraction: float = 0.9 + """The fraction of images to use for training. The remaining images are for eval.""" + depth_unit_scale_factor: float = 1e-3 + """Scales the depth values to meters. Default value is 0.001 for a millimeter to meter conversion.""" + + images_path: Path = Path("images") + """Path to images directory relative to the data path.""" + masks_path: Optional[Path] = None + """Path to masks directory. If not set, masks are not loaded.""" + depths_path: Optional[Path] = None + """Path to depth maps directory. If not set, depths are not loaded.""" + colmap_path: Path = Path("sparse/0") + """Path to the colmap reconstruction directory relative to the data path.""" + + +class ColmapDataParser(DataParser): + """COLMAP DatasetParser. + Expects a folder with the following structure: + images/ # folder containing images used to create the COLMAP model + sparse/0 # folder containing the COLMAP reconstruction (either TEXT or BINARY format) + masks/ # (OPTIONAL) folder containing masks for each image + depths/ # (OPTIONAL) folder containing depth maps for each image + The paths can be different and can be specified in the config. (e.g., sparse/0 -> sparse) + Currently, most COLMAP camera models are supported except for the FULL_OPENCV and THIN_PRISM_FISHEYE models. + + The dataparser loads the downscaled images from folders with `_{downscale_factor}` suffix. + If these folders do not exist, the user can choose to automatically downscale the images and + create these folders. + + The loader is compatible with the datasets processed using the ns-process-data script and + can be used as a drop-in replacement. It further supports datasets like Mip-NeRF 360 (although + in the case of Mip-NeRF 360 the downsampled images may have a different resolution because they + use different rounding when computing the image resolution). + """ + + config: ColmapDataParserConfig + + def __init__(self, config: ColmapDataParserConfig): + super().__init__(config) + self.config = config + self._downscale_factor = None + + def _get_all_images_and_cameras(self, recon_dir: Path): + if (recon_dir / "cameras.txt").exists(): + cam_id_to_camera = colmap_utils.read_cameras_text(recon_dir / "cameras.txt") + im_id_to_image = colmap_utils.read_images_text(recon_dir / "images.txt") + elif (recon_dir / "cameras.bin").exists(): + cam_id_to_camera = colmap_utils.read_cameras_binary(recon_dir / "cameras.bin") + im_id_to_image = colmap_utils.read_images_binary(recon_dir / "images.bin") + else: + raise ValueError(f"Could not find cameras.txt or cameras.bin in {recon_dir}") + + cameras = {} + frames = [] + camera_model = None + + # Parse cameras + for cam_id, cam_data in cam_id_to_camera.items(): + cameras[cam_id] = parse_colmap_camera_params(cam_data) + + # Parse frames + for im_id, im_data in im_id_to_image.items(): + # NB: COLMAP uses Eigen / scalar-first quaternions + # * https://colmap.github.io/format.html + # * https://github.com/colmap/colmap/blob/bf3e19140f491c3042bfd85b7192ef7d249808ec/src/base/pose.cc#L75 + # the `rotation_matrix()` handles that format for us. + rotation = colmap_utils.qvec2rotmat(im_data.qvec) + translation = im_data.tvec.reshape(3, 1) + w2c = np.concatenate([rotation, translation], 1) + w2c = np.concatenate([w2c, np.array([[0, 0, 0, 1]])], 0) + c2w = np.linalg.inv(w2c) + # Convert from COLMAP's camera coordinate system (OpenCV) to ours (OpenGL) + c2w[0:3, 1:3] *= -1 + c2w = c2w[np.array([1, 0, 2, 3]), :] + c2w[2, :] *= -1 + + frame = { + "file_path": (self.config.images_path / im_data.name).as_posix(), + "transform_matrix": c2w, + "colmap_im_id": im_id, + } + frame.update(cameras[im_data.camera_id]) + if self.config.masks_path is not None: + frame["mask_path"] = ((self.config.masks_path / im_data.name).with_suffix(".png").as_posix(),) + if self.config.depths_path is not None: + frame["depth_path"] = ((self.config.depths_path / im_data.name).with_suffix(".png").as_posix(),) + frames.append(frame) + if camera_model is not None: + assert camera_model == frame["camera_model"], "Multiple camera models are not supported" + else: + camera_model = frame["camera_model"] + + out = {} + out["frames"] = frames + applied_transform = np.eye(4)[:3, :] + applied_transform = applied_transform[np.array([1, 0, 2]), :] + applied_transform[2, :] *= -1 + out["applied_transform"] = applied_transform.tolist() + out["camera_model"] = camera_model + assert len(frames) > 0, "No images found in the colmap model" + return out + + def _get_image_indices(self, image_filenames, split): + has_split_files_spec = ( + (self.config.data / "train_list.txt").exists() + or (self.config.data / "test_list.txt").exists() + or (self.config.data / "validation_list.txt").exists() + ) + if (self.config.data / f"{split}_list.txt").exists(): + CONSOLE.log(f"Using {split}_list.txt to get indices for split {split}.") + with (self.config.data / f"{split}_list.txt").open("r", encoding="utf8") as f: + filenames = f.read().splitlines() + # Validate split first + split_filenames = set(self.config.images_path / x for x in filenames) + unmatched_filenames = split_filenames.difference(image_filenames) + if unmatched_filenames: + breakpoint() + raise RuntimeError( + f"Some filenames for split {split} were not found: {set(map(str, unmatched_filenames))}." + ) + + indices = [i for i, path in enumerate(image_filenames) if path in split_filenames] + CONSOLE.log(f"[yellow] Dataset is overriding {split}_indices to {indices}") + indices = np.array(indices, dtype=np.int32) + elif has_split_files_spec: + raise RuntimeError(f"The dataset's list of filenames for split {split} is missing.") + else: + # filter image_filenames and poses based on train/eval split percentage + num_images = len(image_filenames) + num_train_images = math.ceil(num_images * self.config.train_split_fraction) + num_eval_images = num_images - num_train_images + i_all = np.arange(num_images) + i_train = np.linspace( + 0, num_images - 1, num_train_images, dtype=int + ) # equally spaced training images starting and ending at 0 and num_images-1 + i_eval = np.setdiff1d(i_all, i_train) # eval images are the remaining images + assert len(i_eval) == num_eval_images + if split == "train": + indices = i_train + elif split in ["val", "test"]: + indices = i_eval + else: + raise ValueError(f"Unknown dataparser split {split}") + return indices + + def _generate_dataparser_outputs(self, split: str = "train"): + assert self.config.data.exists(), f"Data directory {self.config.data} does not exist." + colmap_path = self.config.data / self.config.colmap_path + assert colmap_path.exists(), f"Colmap path {colmap_path} does not exist." + + meta = self._get_all_images_and_cameras(colmap_path) + camera_type = CAMERA_MODEL_TO_TYPE[meta["camera_model"]] + + image_filenames = [] + mask_filenames = [] + depth_filenames = [] + poses = [] + + fx = [] + fy = [] + cx = [] + cy = [] + height = [] + width = [] + distort = [] + + for frame in meta["frames"]: + fx.append(float(frame["fl_x"])) + fy.append(float(frame["fl_y"])) + cx.append(float(frame["cx"])) + cy.append(float(frame["cy"])) + height.append(int(frame["h"])) + width.append(int(frame["w"])) + distort.append( + camera_utils.get_distortion_params( + k1=float(frame["k1"]) if "k1" in frame else 0.0, + k2=float(frame["k2"]) if "k2" in frame else 0.0, + k3=float(frame["k3"]) if "k3" in frame else 0.0, + k4=float(frame["k4"]) if "k4" in frame else 0.0, + p1=float(frame["p1"]) if "p1" in frame else 0.0, + p2=float(frame["p2"]) if "p2" in frame else 0.0, + ) + ) + + image_filenames.append(Path(frame["file_path"])) + poses.append(frame["transform_matrix"]) + if "mask_path" in frame: + mask_filenames.append(Path(frame["mask_path"])) + if "depth_path" in frame: + depth_filenames.append(Path(frame["depth_path"])) + + assert len(mask_filenames) == 0 or ( + len(mask_filenames) == len(image_filenames) + ), """ + Different number of image and mask filenames. + You should check that mask_path is specified for every frame (or zero frames) in transforms.json. + """ + assert len(depth_filenames) == 0 or ( + len(depth_filenames) == len(image_filenames) + ), """ + Different number of image and depth filenames. + You should check that depth_file_path is specified for every frame (or zero frames) in transforms.json. + """ + poses = torch.from_numpy(np.array(poses).astype(np.float32)) + poses, transform_matrix = camera_utils.auto_orient_and_center_poses( + poses, + method=self.config.orientation_method, + center_method=self.config.center_method, + ) + + # Scale poses + scale_factor = 1.0 + if self.config.auto_scale_poses: + scale_factor /= float(torch.max(torch.abs(poses[:, :3, 3]))) + scale_factor *= self.config.scale_factor + + poses[:, :3, 3] *= scale_factor + + # Choose image_filenames and poses based on split, but after auto orient and scaling the poses. + indices = self._get_image_indices(image_filenames, split) + image_filenames, mask_filenames, depth_filenames, downscale_factor = self._setup_downscale_factor( + image_filenames, mask_filenames, depth_filenames + ) + + image_filenames = [image_filenames[i] for i in indices] + mask_filenames = [mask_filenames[i] for i in indices] if len(mask_filenames) > 0 else [] + depth_filenames = [depth_filenames[i] for i in indices] if len(depth_filenames) > 0 else [] + + idx_tensor = torch.tensor(indices, dtype=torch.long) + poses = poses[idx_tensor] + + # in x,y,z order + # assumes that the scene is centered at the origin + aabb_scale = self.config.scene_scale + scene_box = SceneBox( + aabb=torch.tensor( + [[-aabb_scale, -aabb_scale, -aabb_scale], [aabb_scale, aabb_scale, aabb_scale]], dtype=torch.float32 + ) + ) + + fx = torch.tensor(fx, dtype=torch.float32)[idx_tensor] + fy = torch.tensor(fy, dtype=torch.float32)[idx_tensor] + cx = torch.tensor(cx, dtype=torch.float32)[idx_tensor] + cy = torch.tensor(cy, dtype=torch.float32)[idx_tensor] + height = torch.tensor(height, dtype=torch.int32)[idx_tensor] + width = torch.tensor(width, dtype=torch.int32)[idx_tensor] + distortion_params = torch.stack(distort, dim=0)[idx_tensor] + + cameras = Cameras( + fx=fx, + fy=fy, + cx=cx, + cy=cy, + distortion_params=distortion_params, + height=height, + width=width, + camera_to_worlds=poses[:, :3, :4], + camera_type=camera_type, + ) + + cameras.rescale_output_resolution(scaling_factor=1.0 / downscale_factor) + + if "applied_transform" in meta: + applied_transform = torch.tensor(meta["applied_transform"], dtype=transform_matrix.dtype) + transform_matrix = transform_matrix @ torch.cat( + [applied_transform, torch.tensor([[0, 0, 0, 1]], dtype=transform_matrix.dtype)], 0 + ) + if "applied_scale" in meta: + applied_scale = float(meta["applied_scale"]) + scale_factor *= applied_scale + + dataparser_outputs = DataparserOutputs( + image_filenames=image_filenames, + cameras=cameras, + scene_box=scene_box, + mask_filenames=mask_filenames if len(mask_filenames) > 0 else None, + dataparser_scale=scale_factor, + dataparser_transform=transform_matrix, + metadata={ + "depth_filenames": depth_filenames if len(depth_filenames) > 0 else None, + "depth_unit_scale_factor": self.config.depth_unit_scale_factor, + }, + ) + return dataparser_outputs + + def _setup_downscale_factor( + self, image_filenames: List[Path], mask_filenames: List[Path], depth_filenames: List[Path] + ): + """ + Setup the downscale factor for the dataset. This is used to downscale the images and cameras. + """ + + def get_fname(filepath: Path) -> Path: + """Returns transformed file name when downscale factor is applied""" + parts = list(filepath.parts) + parts[-2] += f"_{self._downscale_factor}" + filepath = Path(*parts) + return self.config.data / filepath + + filepath = next(iter(image_filenames)) + if self._downscale_factor is None: + if self.config.downscale_factor is None: + test_img = Image.open(self.config.data / filepath) + h, w = test_img.size + max_res = max(h, w) + df = 0 + while True: + if (max_res / 2 ** (df)) < MAX_AUTO_RESOLUTION: + break + df += 1 + + self._downscale_factor = 2**df + CONSOLE.log(f"Using image downscale factor of {self._downscale_factor}") + else: + self._downscale_factor = self.config.downscale_factor + if self._downscale_factor > 1 and not all(get_fname(fp).parent.exists() for fp in image_filenames): + # Downscaled images not found + # Ask if user wants to downscale the images automatically here + CONSOLE.print( + f"[bold red]Downscaled images do not exist for factor of {self._downscale_factor}.[/bold red]" + ) + if Confirm.ask("\nWould you like to downscale the images now?", default=False, console=CONSOLE): + # Install the method + image_dir = image_filenames[0].parent + num_downscales = int(math.log2(self._downscale_factor)) + assert 2**num_downscales == self._downscale_factor, "Downscale factor must be a power of 2" + downscale_images(image_dir, num_downscales, folder_name=image_dir.name, nearest_neighbor=False) + if len(mask_filenames) > 0: + mask_dir = mask_filenames[0].parent + downscale_images(mask_dir, num_downscales, folder_name=mask_dir.name, nearest_neighbor=True) + if len(depth_filenames) > 0: + depth_dir = depth_filenames[0].parent + downscale_images(depth_dir, num_downscales, folder_name=depth_dir.name, nearest_neighbor=False) + else: + sys.exit(1) + + # Return transformed filenames + if self._downscale_factor > 1: + image_filenames = [get_fname(fp) for fp in image_filenames] + mask_filenames = [get_fname(fp) for fp in mask_filenames] + depth_filenames = [get_fname(fp) for fp in depth_filenames] + assert isinstance(self._downscale_factor, int) + return image_filenames, mask_filenames, depth_filenames, self._downscale_factor