Jkulhanek/colmap dataparser (#2115)

* Add COLMAP dataparser implementation

* Fix downscale_images input

* Add help to COLMAP dataparser
This commit is contained in:
Jonáš Kulhánek authored and GitHub committed 2023-06-25 17:08:17 +00:00
1 parent ac6a83cc53
commit aa6c052660
2 files changed
+408 -9

No files matched your search

@@ -0,0 +1,403 @@
# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Data parser for nerfstudio datasets. """
from __future__ import annotations
import math
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import List, Literal, Optional, Type
import numpy as np
import torch
from PIL import Image
from rich.prompt import Confirm
from nerfstudio.cameras import camera_utils
from nerfstudio.cameras.cameras import CAMERA_MODEL_TO_TYPE, Cameras
from nerfstudio.data.dataparsers.base_dataparser import DataParser, DataParserConfig, DataparserOutputs
from nerfstudio.data.scene_box import SceneBox
from nerfstudio.data.utils import colmap_parsing_utils as colmap_utils
from nerfstudio.process_data.colmap_utils import parse_colmap_camera_params
from nerfstudio.process_data.process_data_utils import downscale_images
from nerfstudio.utils.rich_utils import CONSOLE
MAX_AUTO_RESOLUTION = 1600
@dataclass
class ColmapDataParserConfig(DataParserConfig):
"""Nerfstudio dataset config"""
_target: Type = field(default_factory=lambda: ColmapDataParser)
"""target class to instantiate"""
data: Path = Path()
"""Directory or explicit json file path specifying location of data."""
scale_factor: float = 1.0
"""How much to scale the camera origins by."""
downscale_factor: Optional[int] = None
"""How much to downscale images. If not set, images are chosen such that the max dimension is <1600px."""
scene_scale: float = 1.0
"""How much to scale the region of interest by."""
orientation_method: Literal["pca", "up", "vertical", "none"] = "up"
"""The method to use for orientation."""
center_method: Literal["poses", "focus", "none"] = "poses"
"""The method to use to center the poses."""
auto_scale_poses: bool = True
"""Whether to automatically scale the poses to fit in +/- 1 bounding box."""
train_split_fraction: float = 0.9
"""The fraction of images to use for training. The remaining images are for eval."""
depth_unit_scale_factor: float = 1e-3
"""Scales the depth values to meters. Default value is 0.001 for a millimeter to meter conversion."""
images_path: Path = Path("images")
"""Path to images directory relative to the data path."""
masks_path: Optional[Path] = None
"""Path to masks directory. If not set, masks are not loaded."""
depths_path: Optional[Path] = None
"""Path to depth maps directory. If not set, depths are not loaded."""
colmap_path: Path = Path("sparse/0")
"""Path to the colmap reconstruction directory relative to the data path."""
class ColmapDataParser(DataParser):
"""COLMAP DatasetParser.
Expects a folder with the following structure:
images/ # folder containing images used to create the COLMAP model
sparse/0 # folder containing the COLMAP reconstruction (either TEXT or BINARY format)
masks/ # (OPTIONAL) folder containing masks for each image
depths/ # (OPTIONAL) folder containing depth maps for each image
The paths can be different and can be specified in the config. (e.g., sparse/0 -> sparse)
Currently, most COLMAP camera models are supported except for the FULL_OPENCV and THIN_PRISM_FISHEYE models.
The dataparser loads the downscaled images from folders with `_{downscale_factor}` suffix.
If these folders do not exist, the user can choose to automatically downscale the images and
create these folders.
The loader is compatible with the datasets processed using the ns-process-data script and
can be used as a drop-in replacement. It further supports datasets like Mip-NeRF 360 (although
in the case of Mip-NeRF 360 the downsampled images may have a different resolution because they
use different rounding when computing the image resolution).
"""
config: ColmapDataParserConfig
def __init__(self, config: ColmapDataParserConfig):
super().__init__(config)
self.config = config
self._downscale_factor = None
def _get_all_images_and_cameras(self, recon_dir: Path):
if (recon_dir / "cameras.txt").exists():
cam_id_to_camera = colmap_utils.read_cameras_text(recon_dir / "cameras.txt")
im_id_to_image = colmap_utils.read_images_text(recon_dir / "images.txt")
elif (recon_dir / "cameras.bin").exists():
cam_id_to_camera = colmap_utils.read_cameras_binary(recon_dir / "cameras.bin")
im_id_to_image = colmap_utils.read_images_binary(recon_dir / "images.bin")
else:
raise ValueError(f"Could not find cameras.txt or cameras.bin in {recon_dir}")
cameras = {}
frames = []
camera_model = None
# Parse cameras
for cam_id, cam_data in cam_id_to_camera.items():
cameras[cam_id] = parse_colmap_camera_params(cam_data)
# Parse frames
for im_id, im_data in im_id_to_image.items():
# NB: COLMAP uses Eigen / scalar-first quaternions
# * https://colmap.github.io/format.html
# * https://github.com/colmap/colmap/blob/bf3e19140f491c3042bfd85b7192ef7d249808ec/src/base/pose.cc#L75
# the `rotation_matrix()` handles that format for us.
rotation = colmap_utils.qvec2rotmat(im_data.qvec)
translation = im_data.tvec.reshape(3, 1)
w2c = np.concatenate([rotation, translation], 1)
w2c = np.concatenate([w2c, np.array([[0, 0, 0, 1]])], 0)
c2w = np.linalg.inv(w2c)
# Convert from COLMAP's camera coordinate system (OpenCV) to ours (OpenGL)
c2w[0:3, 1:3] *= -1
c2w = c2w[np.array([1, 0, 2, 3]), :]
c2w[2, :] *= -1
frame = {
"file_path": (self.config.images_path / im_data.name).as_posix(),
"transform_matrix": c2w,
"colmap_im_id": im_id,
}
frame.update(cameras[im_data.camera_id])
if self.config.masks_path is not None:
frame["mask_path"] = ((self.config.masks_path / im_data.name).with_suffix(".png").as_posix(),)
if self.config.depths_path is not None:
frame["depth_path"] = ((self.config.depths_path / im_data.name).with_suffix(".png").as_posix(),)
frames.append(frame)
if camera_model is not None:
assert camera_model == frame["camera_model"], "Multiple camera models are not supported"
else:
camera_model = frame["camera_model"]
out = {}
out["frames"] = frames
applied_transform = np.eye(4)[:3, :]
applied_transform = applied_transform[np.array([1, 0, 2]), :]
applied_transform[2, :] *= -1
out["applied_transform"] = applied_transform.tolist()
out["camera_model"] = camera_model
assert len(frames) > 0, "No images found in the colmap model"
return out
def _get_image_indices(self, image_filenames, split):
has_split_files_spec = (
(self.config.data / "train_list.txt").exists()
or (self.config.data / "test_list.txt").exists()
or (self.config.data / "validation_list.txt").exists()
)
if (self.config.data / f"{split}_list.txt").exists():
CONSOLE.log(f"Using {split}_list.txt to get indices for split {split}.")
with (self.config.data / f"{split}_list.txt").open("r", encoding="utf8") as f:
filenames = f.read().splitlines()
# Validate split first
split_filenames = set(self.config.images_path / x for x in filenames)
unmatched_filenames = split_filenames.difference(image_filenames)
if unmatched_filenames:
breakpoint()
raise RuntimeError(
f"Some filenames for split {split} were not found: {set(map(str, unmatched_filenames))}."
)
indices = [i for i, path in enumerate(image_filenames) if path in split_filenames]
CONSOLE.log(f"[yellow] Dataset is overriding {split}_indices to {indices}")
indices = np.array(indices, dtype=np.int32)
elif has_split_files_spec:
raise RuntimeError(f"The dataset's list of filenames for split {split} is missing.")
else:
# filter image_filenames and poses based on train/eval split percentage
num_images = len(image_filenames)
num_train_images = math.ceil(num_images * self.config.train_split_fraction)
num_eval_images = num_images - num_train_images
i_all = np.arange(num_images)
i_train = np.linspace(
0, num_images - 1, num_train_images, dtype=int
) # equally spaced training images starting and ending at 0 and num_images-1
i_eval = np.setdiff1d(i_all, i_train) # eval images are the remaining images
assert len(i_eval) == num_eval_images
if split == "train":
indices = i_train
elif split in ["val", "test"]:
indices = i_eval
else:
raise ValueError(f"Unknown dataparser split {split}")
return indices
def _generate_dataparser_outputs(self, split: str = "train"):
assert self.config.data.exists(), f"Data directory {self.config.data} does not exist."
colmap_path = self.config.data / self.config.colmap_path
assert colmap_path.exists(), f"Colmap path {colmap_path} does not exist."
meta = self._get_all_images_and_cameras(colmap_path)
camera_type = CAMERA_MODEL_TO_TYPE[meta["camera_model"]]
image_filenames = []
mask_filenames = []
depth_filenames = []
poses = []
fx = []
fy = []
cx = []
cy = []
height = []
width = []
distort = []
for frame in meta["frames"]:
fx.append(float(frame["fl_x"]))
fy.append(float(frame["fl_y"]))
cx.append(float(frame["cx"]))
cy.append(float(frame["cy"]))
height.append(int(frame["h"]))
width.append(int(frame["w"]))
distort.append(
camera_utils.get_distortion_params(
k1=float(frame["k1"]) if "k1" in frame else 0.0,
k2=float(frame["k2"]) if "k2" in frame else 0.0,
k3=float(frame["k3"]) if "k3" in frame else 0.0,
k4=float(frame["k4"]) if "k4" in frame else 0.0,
p1=float(frame["p1"]) if "p1" in frame else 0.0,
p2=float(frame["p2"]) if "p2" in frame else 0.0,
)
)
image_filenames.append(Path(frame["file_path"]))
poses.append(frame["transform_matrix"])
if "mask_path" in frame:
mask_filenames.append(Path(frame["mask_path"]))
if "depth_path" in frame:
depth_filenames.append(Path(frame["depth_path"]))
assert len(mask_filenames) == 0 or (
len(mask_filenames) == len(image_filenames)
), """
Different number of image and mask filenames.
You should check that mask_path is specified for every frame (or zero frames) in transforms.json.
"""
assert len(depth_filenames) == 0 or (
len(depth_filenames) == len(image_filenames)
), """
Different number of image and depth filenames.
You should check that depth_file_path is specified for every frame (or zero frames) in transforms.json.
"""
poses = torch.from_numpy(np.array(poses).astype(np.float32))
poses, transform_matrix = camera_utils.auto_orient_and_center_poses(
poses,
method=self.config.orientation_method,
center_method=self.config.center_method,
)
# Scale poses
scale_factor = 1.0
if self.config.auto_scale_poses:
scale_factor /= float(torch.max(torch.abs(poses[:, :3, 3])))
scale_factor *= self.config.scale_factor
poses[:, :3, 3] *= scale_factor
# Choose image_filenames and poses based on split, but after auto orient and scaling the poses.
indices = self._get_image_indices(image_filenames, split)
image_filenames, mask_filenames, depth_filenames, downscale_factor = self._setup_downscale_factor(
image_filenames, mask_filenames, depth_filenames
)
image_filenames = [image_filenames[i] for i in indices]
mask_filenames = [mask_filenames[i] for i in indices] if len(mask_filenames) > 0 else []
depth_filenames = [depth_filenames[i] for i in indices] if len(depth_filenames) > 0 else []
idx_tensor = torch.tensor(indices, dtype=torch.long)
poses = poses[idx_tensor]
# in x,y,z order
# assumes that the scene is centered at the origin
aabb_scale = self.config.scene_scale
scene_box = SceneBox(
aabb=torch.tensor(
[[-aabb_scale, -aabb_scale, -aabb_scale], [aabb_scale, aabb_scale, aabb_scale]], dtype=torch.float32
)
)
fx = torch.tensor(fx, dtype=torch.float32)[idx_tensor]
fy = torch.tensor(fy, dtype=torch.float32)[idx_tensor]
cx = torch.tensor(cx, dtype=torch.float32)[idx_tensor]
cy = torch.tensor(cy, dtype=torch.float32)[idx_tensor]
height = torch.tensor(height, dtype=torch.int32)[idx_tensor]
width = torch.tensor(width, dtype=torch.int32)[idx_tensor]
distortion_params = torch.stack(distort, dim=0)[idx_tensor]
cameras = Cameras(
fx=fx,
fy=fy,
cx=cx,
cy=cy,
distortion_params=distortion_params,
height=height,
width=width,
camera_to_worlds=poses[:, :3, :4],
camera_type=camera_type,
)
cameras.rescale_output_resolution(scaling_factor=1.0 / downscale_factor)
if "applied_transform" in meta:
applied_transform = torch.tensor(meta["applied_transform"], dtype=transform_matrix.dtype)
transform_matrix = transform_matrix @ torch.cat(
[applied_transform, torch.tensor([[0, 0, 0, 1]], dtype=transform_matrix.dtype)], 0
)
if "applied_scale" in meta:
applied_scale = float(meta["applied_scale"])
scale_factor *= applied_scale
dataparser_outputs = DataparserOutputs(
image_filenames=image_filenames,
cameras=cameras,
scene_box=scene_box,
mask_filenames=mask_filenames if len(mask_filenames) > 0 else None,
dataparser_scale=scale_factor,
dataparser_transform=transform_matrix,
metadata={
"depth_filenames": depth_filenames if len(depth_filenames) > 0 else None,
"depth_unit_scale_factor": self.config.depth_unit_scale_factor,
},
)
return dataparser_outputs
def _setup_downscale_factor(
self, image_filenames: List[Path], mask_filenames: List[Path], depth_filenames: List[Path]
):
"""
Setup the downscale factor for the dataset. This is used to downscale the images and cameras.
"""
def get_fname(filepath: Path) -> Path:
"""Returns transformed file name when downscale factor is applied"""
parts = list(filepath.parts)
parts[-2] += f"_{self._downscale_factor}"
filepath = Path(*parts)
return self.config.data / filepath
filepath = next(iter(image_filenames))
if self._downscale_factor is None:
if self.config.downscale_factor is None:
test_img = Image.open(self.config.data / filepath)
h, w = test_img.size
max_res = max(h, w)
df = 0
while True:
if (max_res / 2 ** (df)) < MAX_AUTO_RESOLUTION:
break
df += 1
self._downscale_factor = 2**df
CONSOLE.log(f"Using image downscale factor of {self._downscale_factor}")
else:
self._downscale_factor = self.config.downscale_factor
if self._downscale_factor > 1 and not all(get_fname(fp).parent.exists() for fp in image_filenames):
# Downscaled images not found
# Ask if user wants to downscale the images automatically here
CONSOLE.print(
f"[bold red]Downscaled images do not exist for factor of {self._downscale_factor}.[/bold red]"
)
if Confirm.ask("\nWould you like to downscale the images now?", default=False, console=CONSOLE):
# Install the method
image_dir = image_filenames[0].parent
num_downscales = int(math.log2(self._downscale_factor))
assert 2**num_downscales == self._downscale_factor, "Downscale factor must be a power of 2"
downscale_images(image_dir, num_downscales, folder_name=image_dir.name, nearest_neighbor=False)
if len(mask_filenames) > 0:
mask_dir = mask_filenames[0].parent
downscale_images(mask_dir, num_downscales, folder_name=mask_dir.name, nearest_neighbor=True)
if len(depth_filenames) > 0:
depth_dir = depth_filenames[0].parent
downscale_images(depth_dir, num_downscales, folder_name=depth_dir.name, nearest_neighbor=False)
else:
sys.exit(1)
# Return transformed filenames
if self._downscale_factor > 1:
image_filenames = [get_fname(fp) for fp in image_filenames]
mask_filenames = [get_fname(fp) for fp in mask_filenames]
depth_filenames = [get_fname(fp) for fp in depth_filenames]
assert isinstance(self._downscale_factor, int)
return image_filenames, mask_filenames, depth_filenames, self._downscale_factor