From 532f8067fd6fc9300313f597daae88a3d6c01d96 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Fri, 18 Nov 2022 09:19:44 -0800 Subject: [PATCH] Refactor process data (#976) --- .../data/dataparsers/nuscenes_dataparser.py | 2 +- nerfstudio/process_data/__init__.py | 13 + .../{utils => process_data}/colmap_utils.py | 280 ++++++- nerfstudio/process_data/insta360_utils.py | 125 +++ nerfstudio/process_data/polycam_utils.py | 97 +++ nerfstudio/process_data/process_data_utils.py | 211 +++++ nerfstudio/process_data/record3d_utils.py | 94 +++ nerfstudio/utils/rich_utils.py | 18 + nerfstudio/utils/scripts.py | 43 + scripts/process_data.py | 754 +----------------- 10 files changed, 900 insertions(+), 737 deletions(-) create mode 100644 nerfstudio/process_data/__init__.py rename nerfstudio/{utils => process_data}/colmap_utils.py (58%) create mode 100644 nerfstudio/process_data/insta360_utils.py create mode 100644 nerfstudio/process_data/polycam_utils.py create mode 100644 nerfstudio/process_data/process_data_utils.py create mode 100644 nerfstudio/process_data/record3d_utils.py create mode 100644 nerfstudio/utils/scripts.py diff --git a/nerfstudio/data/dataparsers/nuscenes_dataparser.py b/nerfstudio/data/dataparsers/nuscenes_dataparser.py index 88e215611..7a274530d 100644 --- a/nerfstudio/data/dataparsers/nuscenes_dataparser.py +++ b/nerfstudio/data/dataparsers/nuscenes_dataparser.py @@ -31,7 +31,7 @@ from nerfstudio.data.dataparsers.base_dataparser import ( DataparserOutputs, ) from nerfstudio.data.scene_box import SceneBox -from nerfstudio.utils.colmap_utils import qvec2rotmat +from nerfstudio.process_data.colmap_utils import qvec2rotmat def rotation_translation_to_pose(r_vec, t_vec): diff --git a/nerfstudio/process_data/__init__.py b/nerfstudio/process_data/__init__.py new file mode 100644 index 000000000..d926cf95c --- /dev/null +++ b/nerfstudio/process_data/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nerfstudio/utils/colmap_utils.py b/nerfstudio/process_data/colmap_utils.py similarity index 58% rename from nerfstudio/utils/colmap_utils.py rename to nerfstudio/process_data/colmap_utils.py index e58e1448b..c36cc7d5f 100644 --- a/nerfstudio/utils/colmap_utils.py +++ b/nerfstudio/process_data/colmap_utils.py @@ -35,6 +35,7 @@ https://github.com/colmap/colmap/blob/1a4d0bad2e90aa65ce997c9d1779518eaed998d5/s # # Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) +import json import os import struct from dataclasses import dataclass @@ -42,11 +43,22 @@ from io import BufferedReader from pathlib import Path from typing import Dict, Optional, Tuple +import appdirs import numpy as np +import requests +from rich.console import Console +from rich.progress import track +from typing_extensions import Literal + +from nerfstudio.process_data.process_data_utils import CameraModel +from nerfstudio.utils.rich_utils import status +from nerfstudio.utils.scripts import run_command + +CONSOLE = Console(width=120) @dataclass -class CameraModel: +class ColmapCameraModel: """Camera model""" model_id: int @@ -111,21 +123,40 @@ class Point3D: """Point2D indices""" -CAMERA_MODELS = [ - CameraModel(model_id=0, model_name="SIMPLE_PINHOLE", num_params=3), - CameraModel(model_id=1, model_name="PINHOLE", num_params=4), - CameraModel(model_id=2, model_name="SIMPLE_RADIAL", num_params=4), - CameraModel(model_id=3, model_name="RADIAL", num_params=5), - CameraModel(model_id=4, model_name="OPENCV", num_params=8), - CameraModel(model_id=5, model_name="OPENCV_FISHEYE", num_params=8), - CameraModel(model_id=6, model_name="FULL_OPENCV", num_params=12), - CameraModel(model_id=7, model_name="FOV", num_params=5), - CameraModel(model_id=8, model_name="SIMPLE_RADIAL_FISHEYE", num_params=4), - CameraModel(model_id=9, model_name="RADIAL_FISHEYE", num_params=5), - CameraModel(model_id=10, model_name="THIN_PRISM_FISHEYE", num_params=12), +COLMAP_CAMERA_MODELS = [ + ColmapCameraModel(model_id=0, model_name="SIMPLE_PINHOLE", num_params=3), + ColmapCameraModel(model_id=1, model_name="PINHOLE", num_params=4), + ColmapCameraModel(model_id=2, model_name="SIMPLE_RADIAL", num_params=4), + ColmapCameraModel(model_id=3, model_name="RADIAL", num_params=5), + ColmapCameraModel(model_id=4, model_name="OPENCV", num_params=8), + ColmapCameraModel(model_id=5, model_name="OPENCV_FISHEYE", num_params=8), + ColmapCameraModel(model_id=6, model_name="FULL_OPENCV", num_params=12), + ColmapCameraModel(model_id=7, model_name="FOV", num_params=5), + ColmapCameraModel(model_id=8, model_name="SIMPLE_RADIAL_FISHEYE", num_params=4), + ColmapCameraModel(model_id=9, model_name="RADIAL_FISHEYE", num_params=5), + ColmapCameraModel(model_id=10, model_name="THIN_PRISM_FISHEYE", num_params=12), ] -CAMERA_MODEL_IDS = {camera_model.model_id: camera_model for camera_model in CAMERA_MODELS} -CAMERA_MODEL_NAMES = {camera_model.model_name: camera_model for camera_model in CAMERA_MODELS} +COLMAP_CAMERA_MODEL_IDS = {camera_model.model_id: camera_model for camera_model in COLMAP_CAMERA_MODELS} +COLMAP_CAMERA_MODEL_NAMES = {camera_model.model_name: camera_model for camera_model in COLMAP_CAMERA_MODELS} + + +def get_colmap_version(colmap_cmd: str, default_version=3.8) -> float: + """Returns the version of COLMAP. + This code assumes that colmap returns a version string of the form + "COLMAP 3.8 ..." which may not be true for all versions of COLMAP. + + Args: + default_version: Default version to return if COLMAP version can't be determined. + Returns: + The version of COLMAP. + """ + output = run_command(colmap_cmd, verbose=False) + assert output is not None + for line in output.split("\n"): + if line.startswith("COLMAP"): + return float(line.split(" ")[1]) + CONSOLE.print(f"[bold red]Could not find COLMAP version. Using default {default_version}") + return default_version def read_next_bytes(fid: BufferedReader, num_bytes: int, format_char_sequence, endian_character: str = "<"): @@ -183,10 +214,10 @@ def read_cameras_binary(path_to_model_file: Path) -> Dict[int, Camera]: camera_properties = read_next_bytes(fid, num_bytes=24, format_char_sequence="iiQQ") camera_id = camera_properties[0] model_id = camera_properties[1] - model_name = CAMERA_MODEL_IDS[camera_properties[1]].model_name + model_name = COLMAP_CAMERA_MODEL_IDS[camera_properties[1]].model_name width = camera_properties[2] height = camera_properties[3] - num_params = CAMERA_MODEL_IDS[model_id].num_params + num_params = COLMAP_CAMERA_MODEL_IDS[model_id].num_params params = read_next_bytes(fid, num_bytes=8 * num_params, format_char_sequence="d" * num_params) cameras[camera_id] = Camera( id=camera_id, model=model_name, width=width, height=height, params=np.array(params) @@ -430,3 +461,218 @@ def rotmat2qvec(R): if qvec[0] < 0: qvec *= -1 return qvec + + +def get_vocab_tree() -> Path: + """Return path to vocab tree. Downloads vocab tree if it doesn't exist. + + Returns: + The path to the vocab tree. + """ + vocab_tree_filename = Path(appdirs.user_data_dir("nerfstudio")) / "vocab_tree.fbow" + + if not vocab_tree_filename.exists(): + r = requests.get("https://demuc.de/colmap/vocab_tree_flickr100K_words32K.bin", stream=True) + vocab_tree_filename.parent.mkdir(parents=True, exist_ok=True) + with open(vocab_tree_filename, "wb") as f: + total_length = r.headers.get("content-length") + assert total_length is not None + for chunk in track( + r.iter_content(chunk_size=1024), + total=int(total_length) / 1024 + 1, + description="Downloading vocab tree...", + ): + if chunk: + f.write(chunk) + f.flush() + return vocab_tree_filename + + +def run_colmap( + image_dir: Path, + colmap_dir: Path, + camera_model: CameraModel, + gpu: bool = True, + verbose: bool = False, + matching_method: Literal["vocab_tree", "exhaustive", "sequential"] = "vocab_tree", + colmap_cmd: str = "colmap", +) -> None: + """Runs COLMAP on the images. + + Args: + image_dir: Path to the directory containing the images. + colmap_dir: Path to the output directory. + camera_model: Camera model to use. + gpu: If True, use GPU. + verbose: If True, logs the output of the command. + """ + + colmap_version = get_colmap_version(colmap_cmd) + + colmap_database_path = colmap_dir / "database.db" + if colmap_database_path.exists(): + # Can't use missing_ok argument because of Python 3.7 compatibility. + colmap_database_path.unlink() + + # Feature extraction + feature_extractor_cmd = [ + f"{colmap_cmd} feature_extractor", + f"--database_path {colmap_dir / 'database.db'}", + f"--image_path {image_dir}", + "--ImageReader.single_camera 1", + f"--ImageReader.camera_model {camera_model.value}", + f"--SiftExtraction.use_gpu {int(gpu)}", + ] + feature_extractor_cmd = " ".join(feature_extractor_cmd) + with status(msg="[bold yellow]Running COLMAP feature extractor...", spinner="moon", verbose=verbose): + run_command(feature_extractor_cmd, verbose=verbose) + + CONSOLE.log("[bold green]:tada: Done extracting COLMAP features.") + + # Feature matching + feature_matcher_cmd = [ + f"{colmap_cmd} {matching_method}_matcher", + f"--database_path {colmap_dir / 'database.db'}", + f"--SiftMatching.use_gpu {int(gpu)}", + ] + if matching_method == "vocab_tree": + vocab_tree_filename = get_vocab_tree() + feature_matcher_cmd.append(f"--VocabTreeMatching.vocab_tree_path {vocab_tree_filename}") + feature_matcher_cmd = " ".join(feature_matcher_cmd) + with status(msg="[bold yellow]Running COLMAP feature matcher...", spinner="runner", verbose=verbose): + run_command(feature_matcher_cmd, verbose=verbose) + CONSOLE.log("[bold green]:tada: Done matching COLMAP features.") + + # Bundle adjustment + sparse_dir = colmap_dir / "sparse" + sparse_dir.mkdir(parents=True, exist_ok=True) + mapper_cmd = [ + f"{colmap_cmd} mapper", + f"--database_path {colmap_dir / 'database.db'}", + f"--image_path {image_dir}", + f"--output_path {sparse_dir}", + ] + if colmap_version >= 3.7: + mapper_cmd.append("--Mapper.ba_global_function_tolerance 1e-6") + + mapper_cmd = " ".join(mapper_cmd) + + with status( + msg="[bold yellow]Running COLMAP bundle adjustment... (This may take a while)", + spinner="circle", + verbose=verbose, + ): + run_command(mapper_cmd, verbose=verbose) + CONSOLE.log("[bold green]:tada: Done COLMAP bundle adjustment.") + with status(msg="[bold yellow]Refine intrinsics...", spinner="dqpb", verbose=verbose): + bundle_adjuster_cmd = [ + f"{colmap_cmd} bundle_adjuster", + f"--input_path {sparse_dir}/0", + f"--output_path {sparse_dir}/0", + "--BundleAdjustment.refine_principal_point 1", + ] + run_command(" ".join(bundle_adjuster_cmd), verbose=verbose) + CONSOLE.log("[bold green]:tada: Done refining intrinsics.") + + +def colmap_to_json(cameras_path: Path, images_path: Path, output_dir: Path, camera_model: CameraModel) -> int: + """Converts COLMAP's cameras.bin and images.bin to a JSON file. + + Args: + cameras_path: Path to the cameras.bin file. + images_path: Path to the images.bin file. + output_dir: Path to the output directory. + camera_model: Camera model used. + + Returns: + The number of registered images. + """ + + cameras = read_cameras_binary(cameras_path) + images = read_images_binary(images_path) + + # Only supports one camera + camera_params = cameras[1].params + + frames = [] + for _, im_data in images.items(): + rotation = qvec2rotmat(im_data.qvec) + translation = im_data.tvec.reshape(3, 1) + w2c = np.concatenate([rotation, translation], 1) + w2c = np.concatenate([w2c, np.array([[0, 0, 0, 1]])], 0) + c2w = np.linalg.inv(w2c) + # Convert from COLMAP's camera coordinate system to ours + c2w[0:3, 1:3] *= -1 + c2w = c2w[np.array([1, 0, 2, 3]), :] + c2w[2, :] *= -1 + + name = Path(f"./images/{im_data.name}") + + frame = { + "file_path": name.as_posix(), + "transform_matrix": c2w.tolist(), + } + frames.append(frame) + + out = { + "fl_x": float(camera_params[0]), + "fl_y": float(camera_params[1]), + "cx": float(camera_params[2]), + "cy": float(camera_params[3]), + "w": cameras[1].width, + "h": cameras[1].height, + "camera_model": camera_model.value, + } + + if camera_model == CameraModel.OPENCV: + out.update( + { + "k1": float(camera_params[4]), + "k2": float(camera_params[5]), + "p1": float(camera_params[6]), + "p2": float(camera_params[7]), + } + ) + if camera_model == CameraModel.OPENCV_FISHEYE: + out.update( + { + "k1": float(camera_params[4]), + "k2": float(camera_params[5]), + "k3": float(camera_params[6]), + "k4": float(camera_params[7]), + } + ) + + out["frames"] = frames + + with open(output_dir / "transforms.json", "w", encoding="utf-8") as f: + json.dump(out, f, indent=4) + + return len(frames) + + +def get_matching_summary(num_intial_frames: int, num_matched_frames: int) -> str: + """Returns a summary of the matching results. + + Args: + num_intial_frames: The number of initial frames. + num_matched_frames: The number of matched frames. + + Returns: + A summary of the matching results. + """ + match_ratio = num_matched_frames / num_intial_frames + if match_ratio == 1: + return "[bold green]COLAMP found poses for all images, CONGRATS!" + if match_ratio < 0.4: + result = f"[bold red]COLMAP only found poses for {num_matched_frames / num_intial_frames * 100:.2f}%" + result += " of the images. This is low.\nThis can be caused by a variety of reasons," + result += " such poor scene coverage, blurry images, or large exposure changes." + return result + if match_ratio < 0.8: + result = f"[bold yellow]COLMAP only found poses for {num_matched_frames / num_intial_frames * 100:.2f}%" + result += " of the images.\nThis isn't great, but may be ok." + result += "\nMissing poses can be caused by a variety of reasons, such poor scene coverage, blurry images," + result += " or large exposure changes." + return result + return f"[bold green]COLMAP found poses for {num_matched_frames / num_intial_frames * 100:.2f}% of the images." diff --git a/nerfstudio/process_data/insta360_utils.py b/nerfstudio/process_data/insta360_utils.py new file mode 100644 index 000000000..c788836de --- /dev/null +++ b/nerfstudio/process_data/insta360_utils.py @@ -0,0 +1,125 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helper utils for processing insta360 data.""" + +import sys +from pathlib import Path +from typing import List, Tuple + +from rich.console import Console + +from nerfstudio.process_data.process_data_utils import get_num_frames_in_video +from nerfstudio.utils.rich_utils import status +from nerfstudio.utils.scripts import run_command + +CONSOLE = Console(width=120) + + +def get_insta360_filenames(data: Path) -> Tuple[Path, Path]: + """Returns the filenames of the Insta360 videos from a single video file. + + Example input name: VID_20220212_070353_00_003.insv + + Args: + data: Path to a Insta360 file. + + Returns: + The filenames of the Insta360 videios. + """ + if data.suffix != ".insv": + raise ValueError("The input file must be an .insv file.") + file_parts = data.stem.split("_") + + stem_back = f"VID_{file_parts[1]}_{file_parts[2]}_00_{file_parts[4]}.insv" + stem_front = f"VID_{file_parts[1]}_{file_parts[2]}_10_{file_parts[4]}.insv" + + filename_back = data.parent / stem_back + filename_front = data.parent / stem_front + + if not filename_back.exists(): + raise FileNotFoundError(f"Could not find {filename_back}") + if not filename_front.exists(): + raise FileNotFoundError(f"Could not find {filename_front}") + + return filename_back, filename_front + + +def convert_insta360_to_images( + video_front: Path, + video_back: Path, + image_dir: Path, + num_frames_target: int, + crop_percentage: float = 0.7, + verbose: bool = False, +) -> Tuple[List[str], int]: + """Converts a video into a sequence of images. + + Args: + video_front: Path to the front video. + video_back: Path to the back video. + output_dir: Path to the output directory. + num_frames_target: Number of frames to extract. + verbose: If True, logs the output of the command. + Returns: + A tuple containing summary of the conversion and the number of extracted frames. + """ + + with status(msg="Converting video to images...", spinner="bouncingBall", verbose=verbose): + # delete existing images in folder + for img in image_dir.glob("*.png"): + if verbose: + CONSOLE.log(f"Deleting {img}") + img.unlink() + + num_frames_front = get_num_frames_in_video(video_front) + num_frames_back = get_num_frames_in_video(video_back) + if num_frames_front == 0: + CONSOLE.print(f"[bold red]Error: Video has no frames: {video_front}") + sys.exit(1) + if num_frames_back == 0: + CONSOLE.print(f"[bold red]Error: Video has no frames: {video_front}") + sys.exit(1) + + spacing = num_frames_front // (num_frames_target // 2) + vf_cmds = [] + if spacing > 1: + vf_cmds = [f"thumbnail={spacing}", "setpts=N/TB"] + else: + CONSOLE.print("[bold red]Can't satify requested number of frames. Extracting all frames.") + + vf_cmds.append(f"crop=iw*{crop_percentage}:ih*{crop_percentage}") + + front_vf_cmds = vf_cmds + ["transpose=2"] + back_vf_cmds = vf_cmds + ["transpose=1"] + + front_ffmpeg_cmd = f"ffmpeg -i {video_front} -vf {','.join(front_vf_cmds)} -r 1 {image_dir / 'frame_%05d.png'}" + back_ffmpeg_cmd = ( + f"ffmpeg -i {video_back} -vf {','.join(back_vf_cmds)} -r 1 {image_dir / 'back_frame_%05d.png'}" + ) + + run_command(front_ffmpeg_cmd, verbose=verbose) + run_command(back_ffmpeg_cmd, verbose=verbose) + + num_extracted_front_frames = len(list(image_dir.glob("frame*.png"))) + for i, img in enumerate(image_dir.glob("back_frame_*.png")): + img.rename(image_dir / f"frame_{i+1+num_extracted_front_frames:05d}.png") + + num_final_frames = len(list(image_dir.glob("*.png"))) + summary_log = [] + summary_log.append(f"Starting with {num_frames_front + num_frames_back} video frames") + summary_log.append(f"We extracted {num_final_frames} images") + CONSOLE.log("[bold green]:tada: Done converting insta360 to images.") + + return summary_log, num_final_frames diff --git a/nerfstudio/process_data/polycam_utils.py b/nerfstudio/process_data/polycam_utils.py new file mode 100644 index 000000000..51b56667f --- /dev/null +++ b/nerfstudio/process_data/polycam_utils.py @@ -0,0 +1,97 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helper utils for processing polycam data into the nerfstudio format.""" + +import json +import sys +from pathlib import Path +from typing import List + +from rich.console import Console + +from nerfstudio.process_data.process_data_utils import CAMERA_MODELS +from nerfstudio.utils import io + +CONSOLE = Console(width=120) + + +def polycam_to_json( + image_filenames: List[Path], + cameras_dir: Path, + output_dir: Path, + min_blur_score: float = 0.0, + crop_border_pixels: int = 0, +) -> List[str]: + """Convert Polycam data into a nerfstudio dataset. + + Args: + image_filenames: List of paths to the original images. + cameras_dir: Path to the polycam cameras directory. + output_dir: Path to the output directory. + min_blur_score: Minimum blur score to use an image. Images below this value will be skipped. + crop_border_pixels: Number of pixels to crop from each border of the image. + + Returns: + Summary of the conversion. + """ + + frame_json = io.load_from_json(cameras_dir / f"{image_filenames[0].stem}.json") + + # TODO Add per-frame intrinsics + + data = {} + data["fl_x"] = frame_json["fx"] + data["fl_y"] = frame_json["fy"] + data["cx"] = frame_json["cx"] - crop_border_pixels + data["cy"] = frame_json["cy"] - crop_border_pixels + data["w"] = frame_json["width"] - crop_border_pixels * 2 + data["h"] = frame_json["height"] - crop_border_pixels * 2 + data["camera_model"] = CAMERA_MODELS["perspective"].value + # Needs to be a string for camera_utils.auto_orient_and_center_poses + data["orientation_override"] = "none" + + frames = [] + skipped_frames = 0 + for i, image_filename in enumerate(image_filenames): + json_filename = cameras_dir / f"{image_filename.stem}.json" + frame_json = io.load_from_json(json_filename) + if "blur_score" in frame_json and frame_json["blur_score"] < min_blur_score: + skipped_frames += 1 + continue + frame = {} + frame["file_path"] = f"./images/frame_{i+1:05d}{image_filename.suffix}" + # Transform matrix to nerfstudio format. Please refer to the documentation for coordinate system conventions. + frame["transform_matrix"] = [ + [frame_json["t_20"], frame_json["t_21"], frame_json["t_22"], frame_json["t_23"]], + [frame_json["t_00"], frame_json["t_01"], frame_json["t_02"], frame_json["t_03"]], + [frame_json["t_10"], frame_json["t_11"], frame_json["t_12"], frame_json["t_13"]], + [0.0, 0.0, 0.0, 1.0], + ] + frames.append(frame) + data["frames"] = frames + + with open(output_dir / "transforms.json", "w", encoding="utf-8") as f: + json.dump(data, f, indent=4) + + summary = [] + if skipped_frames > 0: + summary.append(f"Skipped {skipped_frames} frames due to low blur score.") + summary.append(f"Final dataset is {len(image_filenames) - skipped_frames} frames.") + + if len(image_filenames) - skipped_frames == 0: + CONSOLE.print("[bold red]No images remain after filtering, exiting") + sys.exit(1) + + return summary diff --git a/nerfstudio/process_data/process_data_utils.py b/nerfstudio/process_data/process_data_utils.py new file mode 100644 index 000000000..106d87f77 --- /dev/null +++ b/nerfstudio/process_data/process_data_utils.py @@ -0,0 +1,211 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helper utils for processing data into the nerfstudio format.""" + +import shutil +import sys +from enum import Enum +from pathlib import Path +from typing import List, Optional, Tuple + +from rich.console import Console + +from nerfstudio.utils.rich_utils import status +from nerfstudio.utils.scripts import run_command + +CONSOLE = Console(width=120) + + +class CameraModel(Enum): + """Enum for camera types.""" + + OPENCV = "OPENCV" + OPENCV_FISHEYE = "OPENCV_FISHEYE" + + +CAMERA_MODELS = { + "perspective": CameraModel.OPENCV, + "fisheye": CameraModel.OPENCV_FISHEYE, +} + + +def get_num_frames_in_video(video: Path) -> int: + """Returns the number of frames in a video. + + Args: + video: Path to a video. + + Returns: + The number of frames in a video. + """ + cmd = f"ffprobe -v error -select_streams v:0 -count_packets \ + -show_entries stream=nb_read_packets -of csv=p=0 {video}" + output = run_command(cmd) + assert output is not None + output = output.strip(" ,\t\n\r") + return int(output) + + +def convert_video_to_images( + video_path: Path, image_dir: Path, num_frames_target: int, verbose: bool = False +) -> Tuple[List[str], int]: + """Converts a video into a sequence of images. + + Args: + video_path: Path to the video. + output_dir: Path to the output directory. + num_frames_target: Number of frames to extract. + verbose: If True, logs the output of the command. + Returns: + A tuple containing summary of the conversion and the number of extracted frames. + """ + + with status(msg="Converting video to images...", spinner="bouncingBall", verbose=verbose): + # delete existing images in folder + for img in image_dir.glob("*.png"): + if verbose: + CONSOLE.log(f"Deleting {img}") + img.unlink() + + num_frames = get_num_frames_in_video(video_path) + if num_frames == 0: + CONSOLE.print(f"[bold red]Error: Video has no frames: {video_path}") + sys.exit(1) + print("Number of frames in video:", num_frames) + + out_filename = image_dir / "frame_%05d.png" + ffmpeg_cmd = f"ffmpeg -i {video_path}" + spacing = num_frames // num_frames_target + + if spacing > 1: + ffmpeg_cmd += f" -vf thumbnail={spacing},setpts=N/TB -r 1" + else: + CONSOLE.print("[bold red]Can't satify requested number of frames. Extracting all frames.") + + ffmpeg_cmd += f" {out_filename}" + + run_command(ffmpeg_cmd, verbose=verbose) + + num_final_frames = len(list(image_dir.glob("*.png"))) + summary_log = [] + summary_log.append(f"Starting with {num_frames} video frames") + summary_log.append(f"We extracted {num_final_frames} images") + CONSOLE.log("[bold green]:tada: Done converting video to images.") + + return summary_log, num_final_frames + + +def copy_images_list( + image_paths: List[Path], image_dir: Path, crop_border_pixels: Optional[int] = None, verbose: bool = False +) -> List[Path]: + """Copy all images in a list of Paths. Useful for filtering from a directory. + Args: + image_paths: List of Paths of images to copy to a new directory. + image_dir: Path to the output directory. + crop_border_pixels: If not None, crops each edge by the specified number of pixels. + verbose: If True, print extra logging. + Returns: + A list of the copied image Paths. + """ + + # Remove original directory only if we provide a proper image folder path + if image_dir.is_dir() and len(image_paths): + shutil.rmtree(image_dir, ignore_errors=True) + image_dir.mkdir(exist_ok=True, parents=True) + + copied_image_paths = [] + + # Images should be 1-indexed for the rest of the pipeline. + for idx, image_path in enumerate(image_paths): + if verbose: + CONSOLE.log(f"Copying image {idx + 1} of {len(image_paths)}...") + copied_image_path = image_dir / f"frame_{idx + 1:05d}{image_path.suffix}" + shutil.copy(image_path, copied_image_path) + copied_image_paths.append(copied_image_path) + + if crop_border_pixels is not None: + file_type = image_paths[0].suffix + filename = f"frame_%05d{file_type}" + crop = f"crop=iw-{crop_border_pixels*2}:ih-{crop_border_pixels*2}" + ffmpeg_cmd = f"ffmpeg -y -i {image_dir / filename} -q:v 2 -vf {crop} {image_dir / filename}" + run_command(ffmpeg_cmd, verbose=verbose) + + num_frames = len(image_paths) + + if num_frames == 0: + CONSOLE.log("[bold red]:skull: No usable images in the data folder.") + else: + CONSOLE.log("[bold green]:tada: Done copying images.") + + return copied_image_paths + + +def copy_images(data: Path, image_dir: Path, verbose) -> int: + """Copy images from a directory to a new directory. + + Args: + data: Path to the directory of images. + image_dir: Path to the output directory. + verbose: If True, print extra logging. + Returns: + The number of images copied. + """ + with status(msg="[bold yellow]Copying images...", spinner="bouncingBall", verbose=verbose): + allowed_exts = [".jpg", ".jpeg", ".png", ".tif", ".tiff"] + image_paths = sorted([p for p in data.glob("[!.]*") if p.suffix.lower() in allowed_exts]) + + num_frames = len(copy_images_list(image_paths, image_dir, verbose)) + + return num_frames + + +def downscale_images(image_dir: Path, num_downscales: int, verbose: bool = False) -> str: + """Downscales the images in the directory. Uses FFMPEG. + + Assumes images are named frame_00001.png, frame_00002.png, etc. + + Args: + image_dir: Path to the directory containing the images. + num_downscales: Number of times to downscale the images. Downscales by 2 each time. + verbose: If True, logs the output of the command. + + Returns: + Summary of downscaling. + """ + + if num_downscales == 0: + return "No downscaling performed." + + with status(msg="[bold yellow]Downscaling images...", spinner="growVertical", verbose=verbose): + downscale_factors = [2**i for i in range(num_downscales + 1)[1:]] + for downscale_factor in downscale_factors: + assert downscale_factor > 1 + assert isinstance(downscale_factor, int) + downscale_dir = image_dir.parent / f"images_{downscale_factor}" + downscale_dir.mkdir(parents=True, exist_ok=True) + file_type = image_dir.glob("frame_*").__next__().suffix + filename = f"frame_%05d{file_type}" + ffmpeg_cmd = [ + f"ffmpeg -i {image_dir / filename} ", + f"-q:v 2 -vf scale=iw/{downscale_factor}:ih/{downscale_factor} ", + f"{downscale_dir / filename}", + ] + ffmpeg_cmd = " ".join(ffmpeg_cmd) + run_command(ffmpeg_cmd, verbose=verbose) + + CONSOLE.log("[bold green]:tada: Done downscaling images.") + downscale_text = [f"[bold blue]{2**(i+1)}x[/bold blue]" for i in range(num_downscales)] + downscale_text = ", ".join(downscale_text[:-1]) + " and " + downscale_text[-1] + return f"We downsampled the images by {downscale_text}" diff --git a/nerfstudio/process_data/record3d_utils.py b/nerfstudio/process_data/record3d_utils.py new file mode 100644 index 000000000..d24292817 --- /dev/null +++ b/nerfstudio/process_data/record3d_utils.py @@ -0,0 +1,94 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helper functions for processing record3d data.""" + +import json +from pathlib import Path +from typing import List + +import numpy as np +from rich.console import Console +from scipy.spatial.transform import Rotation + +from nerfstudio.process_data.process_data_utils import CAMERA_MODELS +from nerfstudio.utils import io + +CONSOLE = Console(width=120) + + +def record3d_to_json(images_paths: List[Path], metadata_path: Path, output_dir: Path, indices: np.ndarray) -> int: + """Converts Record3D's metadata and image paths to a JSON file. + + Args: + images_paths: list if image paths. + metadata_path: Path to the Record3D metadata JSON file. + output_dir: Path to the output directory. + indices: Indices to sample the metadata_path. Should be the same length as images_paths. + + Returns: + The number of registered images. + """ + + assert len(images_paths) == len(indices) + + metadata_dict = io.load_from_json(metadata_path) + + poses_data = np.array(metadata_dict["poses"]) # (N, 3, 4) + camera_to_worlds = np.concatenate( + [Rotation.from_quat(poses_data[:, :4]).as_matrix(), poses_data[:, 4:, None]], + axis=-1, + ).astype(np.float32) + camera_to_worlds = camera_to_worlds[indices] + + homogeneous_coord = np.zeros_like(camera_to_worlds[..., :1, :]) + homogeneous_coord[..., :, 3] = 1 + camera_to_worlds = np.concatenate([camera_to_worlds, homogeneous_coord], -2) + + frames = [] + for i, im_path in enumerate(images_paths): + c2w = camera_to_worlds[i] + frame = { + "file_path": im_path.as_posix(), + "transform_matrix": c2w.tolist(), + } + frames.append(frame) + + # Camera intrinsics + K = np.array(metadata_dict["K"]).reshape((3, 3)).T + focal_length = K[0, 0] + + H = metadata_dict["h"] + W = metadata_dict["w"] + + # TODO(akristoffersen): The metadata dict comes with principle points, + # but caused errors in image coord indexing. Should update once that is fixed. + cx, cy = W / 2, H / 2 + + out = { + "fl_x": focal_length, + "fl_y": focal_length, + "cx": cx, + "cy": cy, + "w": W, + "h": H, + "camera_model": CAMERA_MODELS["perspective"].name, + } + + out["frames"] = frames + + with open(output_dir / "transforms.json", "w", encoding="utf-8") as f: + json.dump(out, f, indent=4) + + return len(frames) diff --git a/nerfstudio/utils/rich_utils.py b/nerfstudio/utils/rich_utils.py index 3a95977bf..d4d6282cf 100644 --- a/nerfstudio/utils/rich_utils.py +++ b/nerfstudio/utils/rich_utils.py @@ -14,9 +14,14 @@ """Additional rich ui components""" +from contextlib import nullcontext + +from rich.console import Console from rich.progress import ProgressColumn from rich.text import Text +CONSOLE = Console(width=120) + class ItersPerSecColumn(ProgressColumn): """Renders the iterations per second for a progress bar.""" @@ -31,3 +36,16 @@ class ItersPerSecColumn(ProgressColumn): if speed is None: return Text("?", style="progress.data.speed") return Text(f"{speed:.2f} {self.suffix}", style="progress.data.speed") + + +def status(msg: str, spinner: str = "bouncingBall", verbose: bool = False): + """A context manager that does nothing is verbose is True. Otherwise it hides logs under a message. + + Args: + msg: The message to log. + spinner: The spinner to use. + verbose: If True, print all logs, else hide them. + """ + if verbose: + return nullcontext() + return CONSOLE.status(msg, spinner=spinner) diff --git a/nerfstudio/utils/scripts.py b/nerfstudio/utils/scripts.py new file mode 100644 index 000000000..54443bae3 --- /dev/null +++ b/nerfstudio/utils/scripts.py @@ -0,0 +1,43 @@ +"""Helpers for running script commands.""" +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +import sys +from typing import Optional + +from rich.console import Console + +CONSOLE = Console(width=120) + + +def run_command(cmd: str, verbose=False) -> Optional[str]: + """Runs a command and returns the output. + + Args: + cmd: Command to run. + verbose: If True, logs the output of the command. + Returns: + The output of the command if return_output is True, otherwise None. + """ + out = subprocess.run(cmd, capture_output=not verbose, shell=True, check=False) + if out.returncode != 0: + CONSOLE.rule("[bold red] :skull: :skull: :skull: ERROR :skull: :skull: :skull: ", style="red") + CONSOLE.print(f"[bold red]Error running command: {cmd}") + CONSOLE.rule(style="red") + CONSOLE.print(out.stderr.decode("utf-8")) + sys.exit(1) + if out.stdout is not None: + return out.stdout.decode("utf-8") + return out diff --git a/scripts/process_data.py b/scripts/process_data.py index 9655d1d35..e726084c3 100755 --- a/scripts/process_data.py +++ b/scripts/process_data.py @@ -1,717 +1,31 @@ #!/usr/bin/env python -# pylint: disable=too-many-lines """Processes a video or image sequence to a nerfstudio compatible dataset.""" -import json -import shutil -import subprocess + import sys import zipfile -from contextlib import nullcontext from dataclasses import dataclass -from enum import Enum from pathlib import Path -from typing import List, Optional, Tuple, Union +from typing import Union -import appdirs import numpy as np -import requests import tyro from rich.console import Console -from rich.progress import track -from scipy.spatial.transform import Rotation from typing_extensions import Annotated, Literal -from nerfstudio.utils import colmap_utils, install_checks, io +from nerfstudio.process_data import ( + colmap_utils, + insta360_utils, + polycam_utils, + process_data_utils, + record3d_utils, +) +from nerfstudio.process_data.process_data_utils import CAMERA_MODELS +from nerfstudio.utils import install_checks CONSOLE = Console(width=120) -class CameraModel(Enum): - """Enum for camera types.""" - - OPENCV = "OPENCV" - OPENCV_FISHEYE = "OPENCV_FISHEYE" - - -CAMERA_MODELS = { - "perspective": CameraModel.OPENCV, - "fisheye": CameraModel.OPENCV_FISHEYE, -} - - -def status(msg: str, spinner: str = "bouncingBall", verbose: bool = False): - """A context manager that does nothing is verbose is True. Otherwise it hides logs under a message. - - Args: - msg: The message to log. - spinner: The spinner to use. - verbose: If True, print all logs, else hide them. - """ - if verbose: - return nullcontext() - return CONSOLE.status(msg, spinner=spinner) - - -def get_colmap_version(colmap_cmd: str, default_version=3.8) -> float: - """Returns the version of COLMAP. - This code assumes that colmap returns a version string of the form - "COLMAP 3.8 ..." which may not be true for all versions of COLMAP. - - Args: - default_version: Default version to return if COLMAP version can't be determined. - Returns: - The version of COLMAP. - """ - output = run_command(colmap_cmd, verbose=False) - assert output is not None - for line in output.split("\n"): - if line.startswith("COLMAP"): - return float(line.split(" ")[1]) - CONSOLE.print(f"[bold red]Could not find COLMAP version. Using default {default_version}") - return default_version - - -def get_vocab_tree() -> Path: - """Return path to vocab tree. Downloads vocab tree if it doesn't exist. - - Returns: - The path to the vocab tree. - """ - vocab_tree_filename = Path(appdirs.user_data_dir("nerfstudio")) / "vocab_tree.fbow" - - if not vocab_tree_filename.exists(): - r = requests.get("https://demuc.de/colmap/vocab_tree_flickr100K_words32K.bin", stream=True) - vocab_tree_filename.parent.mkdir(parents=True, exist_ok=True) - with open(vocab_tree_filename, "wb") as f: - total_length = r.headers.get("content-length") - assert total_length is not None - for chunk in track( - r.iter_content(chunk_size=1024), - total=int(total_length) / 1024 + 1, - description="Downloading vocab tree...", - ): - if chunk: - f.write(chunk) - f.flush() - return vocab_tree_filename - - -def run_command(cmd: str, verbose=False) -> Optional[str]: - """Runs a command and returns the output. - - Args: - cmd: Command to run. - verbose: If True, logs the output of the command. - Returns: - The output of the command if return_output is True, otherwise None. - """ - out = subprocess.run(cmd, capture_output=not verbose, shell=True, check=False) - if out.returncode != 0: - CONSOLE.rule("[bold red] :skull: :skull: :skull: ERROR :skull: :skull: :skull: ", style="red") - CONSOLE.print(f"[bold red]Error running command: {cmd}") - CONSOLE.rule(style="red") - CONSOLE.print(out.stderr.decode("utf-8")) - sys.exit(1) - if out.stdout is not None: - return out.stdout.decode("utf-8") - return out - - -def get_num_frames_in_video(video: Path) -> int: - """Returns the number of frames in a video. - - Args: - video: Path to a video. - - Returns: - The number of frames in a video. - """ - cmd = f"ffprobe -v error -select_streams v:0 -count_packets \ - -show_entries stream=nb_read_packets -of csv=p=0 {video}" - output = run_command(cmd) - assert output is not None - output = output.strip(" ,\t\n\r") - return int(output) - - -def convert_video_to_images( - video_path: Path, image_dir: Path, num_frames_target: int, verbose: bool = False -) -> Tuple[List[str], int]: - """Converts a video into a sequence of images. - - Args: - video_path: Path to the video. - output_dir: Path to the output directory. - num_frames_target: Number of frames to extract. - verbose: If True, logs the output of the command. - Returns: - A tuple containing summary of the conversion and the number of extracted frames. - """ - - with status(msg="Converting video to images...", spinner="bouncingBall", verbose=verbose): - # delete existing images in folder - for img in image_dir.glob("*.png"): - if verbose: - CONSOLE.log(f"Deleting {img}") - img.unlink() - - num_frames = get_num_frames_in_video(video_path) - if num_frames == 0: - CONSOLE.print(f"[bold red]Error: Video has no frames: {video_path}") - sys.exit(1) - print("Number of frames in video:", num_frames) - - out_filename = image_dir / "frame_%05d.png" - ffmpeg_cmd = f"ffmpeg -i {video_path}" - spacing = num_frames // num_frames_target - - if spacing > 1: - ffmpeg_cmd += f" -vf thumbnail={spacing},setpts=N/TB -r 1" - else: - CONSOLE.print("[bold red]Can't satify requested number of frames. Extracting all frames.") - - ffmpeg_cmd += f" {out_filename}" - - run_command(ffmpeg_cmd, verbose=verbose) - - num_final_frames = len(list(image_dir.glob("*.png"))) - summary_log = [] - summary_log.append(f"Starting with {num_frames} video frames") - summary_log.append(f"We extracted {num_final_frames} images") - CONSOLE.log("[bold green]:tada: Done converting video to images.") - - return summary_log, num_final_frames - - -def convert_insta360_to_images( - video_front: Path, - video_back: Path, - image_dir: Path, - num_frames_target: int, - crop_percentage: float = 0.7, - verbose: bool = False, -) -> Tuple[List[str], int]: - """Converts a video into a sequence of images. - - Args: - video_front: Path to the front video. - video_back: Path to the back video. - output_dir: Path to the output directory. - num_frames_target: Number of frames to extract. - verbose: If True, logs the output of the command. - Returns: - A tuple containing summary of the conversion and the number of extracted frames. - """ - - with status(msg="Converting video to images...", spinner="bouncingBall", verbose=verbose): - # delete existing images in folder - for img in image_dir.glob("*.png"): - if verbose: - CONSOLE.log(f"Deleting {img}") - img.unlink() - - num_frames_front = get_num_frames_in_video(video_front) - num_frames_back = get_num_frames_in_video(video_back) - if num_frames_front == 0: - CONSOLE.print(f"[bold red]Error: Video has no frames: {video_front}") - sys.exit(1) - if num_frames_back == 0: - CONSOLE.print(f"[bold red]Error: Video has no frames: {video_front}") - sys.exit(1) - - spacing = num_frames_front // (num_frames_target // 2) - vf_cmds = [] - if spacing > 1: - vf_cmds = [f"thumbnail={spacing}", "setpts=N/TB"] - else: - CONSOLE.print("[bold red]Can't satify requested number of frames. Extracting all frames.") - - vf_cmds.append(f"crop=iw*{crop_percentage}:ih*{crop_percentage}") - - front_vf_cmds = vf_cmds + ["transpose=2"] - back_vf_cmds = vf_cmds + ["transpose=1"] - - front_ffmpeg_cmd = f"ffmpeg -i {video_front} -vf {','.join(front_vf_cmds)} -r 1 {image_dir / 'frame_%05d.png'}" - back_ffmpeg_cmd = ( - f"ffmpeg -i {video_back} -vf {','.join(back_vf_cmds)} -r 1 {image_dir / 'back_frame_%05d.png'}" - ) - - run_command(front_ffmpeg_cmd, verbose=verbose) - run_command(back_ffmpeg_cmd, verbose=verbose) - - num_extracted_front_frames = len(list(image_dir.glob("frame*.png"))) - for i, img in enumerate(image_dir.glob("back_frame_*.png")): - img.rename(image_dir / f"frame_{i+1+num_extracted_front_frames:05d}.png") - - num_final_frames = len(list(image_dir.glob("*.png"))) - summary_log = [] - summary_log.append(f"Starting with {num_frames_front + num_frames_back} video frames") - summary_log.append(f"We extracted {num_final_frames} images") - CONSOLE.log("[bold green]:tada: Done converting insta360 to images.") - - return summary_log, num_final_frames - - -def copy_images_list( - image_paths: List[Path], image_dir: Path, crop_border_pixels: Optional[int] = None, verbose: bool = False -) -> List[Path]: - """Copy all images in a list of Paths. Useful for filtering from a directory. - Args: - image_paths: List of Paths of images to copy to a new directory. - image_dir: Path to the output directory. - crop_border_pixels: If not None, crops each edge by the specified number of pixels. - verbose: If True, print extra logging. - Returns: - A list of the copied image Paths. - """ - - # Remove original directory only if we provide a proper image folder path - if image_dir.is_dir() and len(image_paths): - shutil.rmtree(image_dir, ignore_errors=True) - image_dir.mkdir(exist_ok=True, parents=True) - - copied_image_paths = [] - - # Images should be 1-indexed for the rest of the pipeline. - for idx, image_path in enumerate(image_paths): - if verbose: - CONSOLE.log(f"Copying image {idx + 1} of {len(image_paths)}...") - copied_image_path = image_dir / f"frame_{idx + 1:05d}{image_path.suffix}" - shutil.copy(image_path, copied_image_path) - copied_image_paths.append(copied_image_path) - - if crop_border_pixels is not None: - file_type = image_paths[0].suffix - filename = f"frame_%05d{file_type}" - crop = f"crop=iw-{crop_border_pixels*2}:ih-{crop_border_pixels*2}" - ffmpeg_cmd = f"ffmpeg -y -i {image_dir / filename} -q:v 2 -vf {crop} {image_dir / filename}" - run_command(ffmpeg_cmd, verbose=verbose) - - num_frames = len(image_paths) - - if num_frames == 0: - CONSOLE.log("[bold red]:skull: No usable images in the data folder.") - else: - CONSOLE.log("[bold green]:tada: Done copying images.") - - return copied_image_paths - - -def copy_images(data: Path, image_dir: Path, verbose) -> int: - """Copy images from a directory to a new directory. - - Args: - data: Path to the directory of images. - image_dir: Path to the output directory. - verbose: If True, print extra logging. - Returns: - The number of images copied. - """ - with status(msg="[bold yellow]Copying images...", spinner="bouncingBall", verbose=verbose): - allowed_exts = [".jpg", ".jpeg", ".png", ".tif", ".tiff"] - image_paths = sorted([p for p in data.glob("[!.]*") if p.suffix.lower() in allowed_exts]) - - num_frames = len(copy_images_list(image_paths, image_dir, verbose)) - - return num_frames - - -def downscale_images(image_dir: Path, num_downscales: int, verbose: bool = False) -> str: - """Downscales the images in the directory. Uses FFMPEG. - - Assumes images are named frame_00001.png, frame_00002.png, etc. - - Args: - image_dir: Path to the directory containing the images. - num_downscales: Number of times to downscale the images. Downscales by 2 each time. - verbose: If True, logs the output of the command. - - Returns: - Summary of downscaling. - """ - - if num_downscales == 0: - return "No downscaling performed." - - with status(msg="[bold yellow]Downscaling images...", spinner="growVertical", verbose=verbose): - downscale_factors = [2**i for i in range(num_downscales + 1)[1:]] - for downscale_factor in downscale_factors: - assert downscale_factor > 1 - assert isinstance(downscale_factor, int) - downscale_dir = image_dir.parent / f"images_{downscale_factor}" - downscale_dir.mkdir(parents=True, exist_ok=True) - file_type = image_dir.glob("frame_*").__next__().suffix - filename = f"frame_%05d{file_type}" - ffmpeg_cmd = [ - f"ffmpeg -i {image_dir / filename} ", - f"-q:v 2 -vf scale=iw/{downscale_factor}:ih/{downscale_factor} ", - f"{downscale_dir / filename}", - ] - ffmpeg_cmd = " ".join(ffmpeg_cmd) - run_command(ffmpeg_cmd, verbose=verbose) - - CONSOLE.log("[bold green]:tada: Done downscaling images.") - downscale_text = [f"[bold blue]{2**(i+1)}x[/bold blue]" for i in range(num_downscales)] - downscale_text = ", ".join(downscale_text[:-1]) + " and " + downscale_text[-1] - return f"We downsampled the images by {downscale_text}" - - -def run_colmap( - image_dir: Path, - colmap_dir: Path, - camera_model: CameraModel, - gpu: bool = True, - verbose: bool = False, - matching_method: Literal["vocab_tree", "exhaustive", "sequential"] = "vocab_tree", - colmap_cmd: str = "colmap", -) -> None: - """Runs COLMAP on the images. - - Args: - image_dir: Path to the directory containing the images. - colmap_dir: Path to the output directory. - camera_model: Camera model to use. - gpu: If True, use GPU. - verbose: If True, logs the output of the command. - """ - - colmap_version = get_colmap_version(colmap_cmd) - - colmap_database_path = colmap_dir / "database.db" - if colmap_database_path.exists(): - # Can't use missing_ok argument because of Python 3.7 compatibility. - colmap_database_path.unlink() - - # Feature extraction - feature_extractor_cmd = [ - f"{colmap_cmd} feature_extractor", - f"--database_path {colmap_dir / 'database.db'}", - f"--image_path {image_dir}", - "--ImageReader.single_camera 1", - f"--ImageReader.camera_model {camera_model.value}", - f"--SiftExtraction.use_gpu {int(gpu)}", - ] - feature_extractor_cmd = " ".join(feature_extractor_cmd) - with status(msg="[bold yellow]Running COLMAP feature extractor...", spinner="moon", verbose=verbose): - run_command(feature_extractor_cmd, verbose=verbose) - - CONSOLE.log("[bold green]:tada: Done extracting COLMAP features.") - - # Feature matching - feature_matcher_cmd = [ - f"{colmap_cmd} {matching_method}_matcher", - f"--database_path {colmap_dir / 'database.db'}", - f"--SiftMatching.use_gpu {int(gpu)}", - ] - if matching_method == "vocab_tree": - vocab_tree_filename = get_vocab_tree() - feature_matcher_cmd.append(f"--VocabTreeMatching.vocab_tree_path {vocab_tree_filename}") - feature_matcher_cmd = " ".join(feature_matcher_cmd) - with status(msg="[bold yellow]Running COLMAP feature matcher...", spinner="runner", verbose=verbose): - run_command(feature_matcher_cmd, verbose=verbose) - CONSOLE.log("[bold green]:tada: Done matching COLMAP features.") - - # Bundle adjustment - sparse_dir = colmap_dir / "sparse" - sparse_dir.mkdir(parents=True, exist_ok=True) - mapper_cmd = [ - f"{colmap_cmd} mapper", - f"--database_path {colmap_dir / 'database.db'}", - f"--image_path {image_dir}", - f"--output_path {sparse_dir}", - ] - if colmap_version >= 3.7: - mapper_cmd.append("--Mapper.ba_global_function_tolerance 1e-6") - - mapper_cmd = " ".join(mapper_cmd) - - with status( - msg="[bold yellow]Running COLMAP bundle adjustment... (This may take a while)", - spinner="circle", - verbose=verbose, - ): - run_command(mapper_cmd, verbose=verbose) - CONSOLE.log("[bold green]:tada: Done COLMAP bundle adjustment.") - with status(msg="[bold yellow]Refine intrinsics...", spinner="dqpb", verbose=verbose): - bundle_adjuster_cmd = [ - f"{colmap_cmd} bundle_adjuster", - f"--input_path {sparse_dir}/0", - f"--output_path {sparse_dir}/0", - "--BundleAdjustment.refine_principal_point 1", - ] - run_command(" ".join(bundle_adjuster_cmd), verbose=verbose) - CONSOLE.log("[bold green]:tada: Done refining intrinsics.") - - -def colmap_to_json(cameras_path: Path, images_path: Path, output_dir: Path, camera_model: CameraModel) -> int: - """Converts COLMAP's cameras.bin and images.bin to a JSON file. - - Args: - cameras_path: Path to the cameras.bin file. - images_path: Path to the images.bin file. - output_dir: Path to the output directory. - camera_model: Camera model used. - - Returns: - The number of registered images. - """ - - cameras = colmap_utils.read_cameras_binary(cameras_path) - images = colmap_utils.read_images_binary(images_path) - - # Only supports one camera - camera_params = cameras[1].params - - frames = [] - for _, im_data in images.items(): - rotation = colmap_utils.qvec2rotmat(im_data.qvec) - translation = im_data.tvec.reshape(3, 1) - w2c = np.concatenate([rotation, translation], 1) - w2c = np.concatenate([w2c, np.array([[0, 0, 0, 1]])], 0) - c2w = np.linalg.inv(w2c) - # Convert from COLMAP's camera coordinate system to ours - c2w[0:3, 1:3] *= -1 - c2w = c2w[np.array([1, 0, 2, 3]), :] - c2w[2, :] *= -1 - - name = Path(f"./images/{im_data.name}") - - frame = { - "file_path": name.as_posix(), - "transform_matrix": c2w.tolist(), - } - frames.append(frame) - - out = { - "fl_x": float(camera_params[0]), - "fl_y": float(camera_params[1]), - "cx": float(camera_params[2]), - "cy": float(camera_params[3]), - "w": cameras[1].width, - "h": cameras[1].height, - "camera_model": camera_model.value, - } - - if camera_model == CameraModel.OPENCV: - out.update( - { - "k1": float(camera_params[4]), - "k2": float(camera_params[5]), - "p1": float(camera_params[6]), - "p2": float(camera_params[7]), - } - ) - if camera_model == CameraModel.OPENCV_FISHEYE: - out.update( - { - "k1": float(camera_params[4]), - "k2": float(camera_params[5]), - "k3": float(camera_params[6]), - "k4": float(camera_params[7]), - } - ) - - out["frames"] = frames - - with open(output_dir / "transforms.json", "w", encoding="utf-8") as f: - json.dump(out, f, indent=4) - - return len(frames) - - -def record3d_to_json(images_paths: List[Path], metadata_path: Path, output_dir: Path, indices: np.ndarray) -> int: - """Converts Record3D's metadata and image paths to a JSON file. - - Args: - images_paths: list if image paths. - metadata_path: Path to the Record3D metadata JSON file. - output_dir: Path to the output directory. - indices: Indices to sample the metadata_path. Should be the same length as images_paths. - - Returns: - The number of registered images. - """ - - assert len(images_paths) == len(indices) - - metadata_dict = io.load_from_json(metadata_path) - - poses_data = np.array(metadata_dict["poses"]) # (N, 3, 4) - camera_to_worlds = np.concatenate( - [Rotation.from_quat(poses_data[:, :4]).as_matrix(), poses_data[:, 4:, None]], - axis=-1, - ).astype(np.float32) - camera_to_worlds = camera_to_worlds[indices] - - homogeneous_coord = np.zeros_like(camera_to_worlds[..., :1, :]) - homogeneous_coord[..., :, 3] = 1 - camera_to_worlds = np.concatenate([camera_to_worlds, homogeneous_coord], -2) - - frames = [] - for i, im_path in enumerate(images_paths): - c2w = camera_to_worlds[i] - frame = { - "file_path": im_path.as_posix(), - "transform_matrix": c2w.tolist(), - } - frames.append(frame) - - # Camera intrinsics - K = np.array(metadata_dict["K"]).reshape((3, 3)).T - focal_length = K[0, 0] - - H = metadata_dict["h"] - W = metadata_dict["w"] - - # TODO(akristoffersen): The metadata dict comes with principle points, - # but caused errors in image coord indexing. Should update once that is fixed. - cx, cy = W / 2, H / 2 - - out = { - "fl_x": focal_length, - "fl_y": focal_length, - "cx": cx, - "cy": cy, - "w": W, - "h": H, - "camera_model": CAMERA_MODELS["perspective"].name, - } - - out["frames"] = frames - - with open(output_dir / "transforms.json", "w", encoding="utf-8") as f: - json.dump(out, f, indent=4) - - return len(frames) - - -def get_insta360_filenames(data: Path) -> Tuple[Path, Path]: - """Returns the filenames of the Insta360 videos from a single video file. - - Example input name: VID_20220212_070353_00_003.insv - - Args: - data: Path to a Insta360 file. - - Returns: - The filenames of the Insta360 videios. - """ - if data.suffix != ".insv": - raise ValueError("The input file must be an .insv file.") - file_parts = data.stem.split("_") - - stem_back = f"VID_{file_parts[1]}_{file_parts[2]}_00_{file_parts[4]}.insv" - stem_front = f"VID_{file_parts[1]}_{file_parts[2]}_10_{file_parts[4]}.insv" - - filename_back = data.parent / stem_back - filename_front = data.parent / stem_front - - if not filename_back.exists(): - raise FileNotFoundError(f"Could not find {filename_back}") - if not filename_front.exists(): - raise FileNotFoundError(f"Could not find {filename_front}") - - return filename_back, filename_front - - -def get_matching_summary(num_intial_frames: int, num_matched_frames: int) -> str: - """Returns a summary of the matching results. - - Args: - num_intial_frames: The number of initial frames. - num_matched_frames: The number of matched frames. - - Returns: - A summary of the matching results. - """ - match_ratio = num_matched_frames / num_intial_frames - if match_ratio == 1: - return "[bold green]COLAMP found poses for all images, CONGRATS!" - if match_ratio < 0.4: - result = f"[bold red]COLMAP only found poses for {num_matched_frames / num_intial_frames * 100:.2f}%" - result += " of the images. This is low.\nThis can be caused by a variety of reasons," - result += " such poor scene coverage, blurry images, or large exposure changes." - return result - if match_ratio < 0.8: - result = f"[bold yellow]COLMAP only found poses for {num_matched_frames / num_intial_frames * 100:.2f}%" - result += " of the images.\nThis isn't great, but may be ok." - result += "\nMissing poses can be caused by a variety of reasons, such poor scene coverage, blurry images," - result += " or large exposure changes." - return result - return f"[bold green]COLMAP found poses for {num_matched_frames / num_intial_frames * 100:.2f}% of the images." - - -def polycam_to_json( - image_filenames: List[Path], - cameras_dir: Path, - output_dir: Path, - min_blur_score: float = 0.0, - crop_border_pixels: int = 0, -) -> List[str]: - """Convert Polycam data into a nerfstudio dataset. - - Args: - image_filenames: List of paths to the original images. - cameras_dir: Path to the polycam cameras directory. - output_dir: Path to the output directory. - min_blur_score: Minimum blur score to use an image. Images below this value will be skipped. - crop_border_pixels: Number of pixels to crop from each border of the image. - - Returns: - Summary of the conversion. - """ - - frame_json = io.load_from_json(cameras_dir / f"{image_filenames[0].stem}.json") - - # TODO Add per-frame intrinsics - - data = {} - data["fl_x"] = frame_json["fx"] - data["fl_y"] = frame_json["fy"] - data["cx"] = frame_json["cx"] - crop_border_pixels - data["cy"] = frame_json["cy"] - crop_border_pixels - data["w"] = frame_json["width"] - crop_border_pixels * 2 - data["h"] = frame_json["height"] - crop_border_pixels * 2 - data["camera_model"] = CAMERA_MODELS["perspective"].value - # Needs to be a string for camera_utils.auto_orient_and_center_poses - data["orientation_override"] = "none" - - frames = [] - skipped_frames = 0 - for i, image_filename in enumerate(image_filenames): - json_filename = cameras_dir / f"{image_filename.stem}.json" - frame_json = io.load_from_json(json_filename) - if "blur_score" in frame_json and frame_json["blur_score"] < min_blur_score: - skipped_frames += 1 - continue - frame = {} - frame["file_path"] = f"./images/frame_{i+1:05d}{image_filename.suffix}" - # Transform matrix to nerfstudio format. Please refer to the documentation for coordinate system conventions. - frame["transform_matrix"] = [ - [frame_json["t_20"], frame_json["t_21"], frame_json["t_22"], frame_json["t_23"]], - [frame_json["t_00"], frame_json["t_01"], frame_json["t_02"], frame_json["t_03"]], - [frame_json["t_10"], frame_json["t_11"], frame_json["t_12"], frame_json["t_13"]], - [0.0, 0.0, 0.0, 1.0], - ] - frames.append(frame) - data["frames"] = frames - - with open(output_dir / "transforms.json", "w", encoding="utf-8") as f: - json.dump(data, f, indent=4) - - summary = [] - if skipped_frames > 0: - summary.append(f"Skipped {skipped_frames} frames due to low blur score.") - summary.append(f"Final dataset is {len(image_filenames) - skipped_frames} frames.") - - if len(image_filenames) - skipped_frames == 0: - CONSOLE.print("[bold red]No images remain after filtering, exiting") - sys.exit(1) - - return summary - - @dataclass class ProcessImages: """Process images into a nerfstudio dataset. @@ -755,18 +69,18 @@ class ProcessImages: summary_log = [] # Copy images to output directory - num_frames = copy_images(self.data, image_dir=image_dir, verbose=self.verbose) + num_frames = process_data_utils.copy_images(self.data, image_dir=image_dir, verbose=self.verbose) summary_log.append(f"Starting with {num_frames} images") # Downscale images - summary_log.append(downscale_images(image_dir, self.num_downscales, verbose=self.verbose)) + summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose)) # Run COLMAP colmap_dir = self.output_dir / "colmap" if not self.skip_colmap: colmap_dir.mkdir(parents=True, exist_ok=True) - run_colmap( + colmap_utils.run_colmap( image_dir=image_dir, colmap_dir=colmap_dir, camera_model=CAMERA_MODELS[self.camera_type], @@ -779,14 +93,14 @@ class ProcessImages: # Save transforms.json if (colmap_dir / "sparse" / "0" / "cameras.bin").exists(): with CONSOLE.status("[bold yellow]Saving results to transforms.json", spinner="balloon"): - num_matched_frames = colmap_to_json( + num_matched_frames = colmap_utils.colmap_to_json( cameras_path=colmap_dir / "sparse" / "0" / "cameras.bin", images_path=colmap_dir / "sparse" / "0" / "images.bin", output_dir=self.output_dir, camera_model=CAMERA_MODELS[self.camera_type], ) summary_log.append(f"Colmap matched {num_matched_frames} images") - summary_log.append(get_matching_summary(num_frames, num_matched_frames)) + summary_log.append(colmap_utils.get_matching_summary(num_frames, num_matched_frames)) else: CONSOLE.log("[bold yellow]Warning: could not find existing COLMAP results. Not generating transforms.json") @@ -841,19 +155,19 @@ class ProcessVideo: image_dir.mkdir(parents=True, exist_ok=True) # Convert video to images - summary_log, num_extracted_frames = convert_video_to_images( + summary_log, num_extracted_frames = process_data_utils.convert_video_to_images( self.data, image_dir=image_dir, num_frames_target=self.num_frames_target, verbose=self.verbose ) # Downscale images - summary_log.append(downscale_images(image_dir, self.num_downscales, verbose=self.verbose)) + summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose)) # Run Colmap colmap_dir = self.output_dir / "colmap" if not self.skip_colmap: colmap_dir.mkdir(parents=True, exist_ok=True) - run_colmap( + colmap_utils.run_colmap( image_dir=image_dir, colmap_dir=colmap_dir, camera_model=CAMERA_MODELS[self.camera_type], @@ -866,14 +180,14 @@ class ProcessVideo: # Save transforms.json if (colmap_dir / "sparse" / "0" / "cameras.bin").exists(): with CONSOLE.status("[bold yellow]Saving results to transforms.json", spinner="balloon"): - num_matched_frames = colmap_to_json( + num_matched_frames = colmap_utils.colmap_to_json( cameras_path=colmap_dir / "sparse" / "0" / "cameras.bin", images_path=colmap_dir / "sparse" / "0" / "images.bin", output_dir=self.output_dir, camera_model=CAMERA_MODELS[self.camera_type], ) summary_log.append(f"Colmap matched {num_matched_frames} images") - summary_log.append(get_matching_summary(num_extracted_frames, num_matched_frames)) + summary_log.append(colmap_utils.get_matching_summary(num_extracted_frames, num_matched_frames)) else: CONSOLE.log("[bold yellow]Warning: could not find existing COLMAP results. Not generating transforms.json") @@ -929,10 +243,10 @@ class ProcessInsta360: image_dir = self.output_dir / "images" image_dir.mkdir(parents=True, exist_ok=True) - filename_back, filename_front = get_insta360_filenames(self.data) + filename_back, filename_front = insta360_utils.get_insta360_filenames(self.data) # Convert video to images - summary_log, num_extracted_frames = convert_insta360_to_images( + summary_log, num_extracted_frames = insta360_utils.convert_insta360_to_images( video_front=filename_front, video_back=filename_back, image_dir=image_dir, @@ -941,14 +255,14 @@ class ProcessInsta360: ) # Downscale images - summary_log.append(downscale_images(image_dir, self.num_downscales, verbose=self.verbose)) + summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose)) # Run Colmap colmap_dir = self.output_dir / "colmap" if not self.skip_colmap: colmap_dir.mkdir(parents=True, exist_ok=True) - run_colmap( + colmap_utils.run_colmap( image_dir=image_dir, colmap_dir=colmap_dir, camera_model=CAMERA_MODELS["fisheye"], @@ -961,14 +275,14 @@ class ProcessInsta360: # Save transforms.json if (colmap_dir / "sparse" / "0" / "cameras.bin").exists(): with CONSOLE.status("[bold yellow]Saving results to transforms.json", spinner="balloon"): - num_matched_frames = colmap_to_json( + num_matched_frames = colmap_utils.colmap_to_json( cameras_path=colmap_dir / "sparse" / "0" / "cameras.bin", images_path=colmap_dir / "sparse" / "0" / "images.bin", output_dir=self.output_dir, camera_model=CAMERA_MODELS["fisheye"], ) summary_log.append(f"Colmap matched {num_matched_frames} images") - summary_log.append(get_matching_summary(num_extracted_frames, num_matched_frames)) + summary_log.append(colmap_utils.get_matching_summary(num_extracted_frames, num_matched_frames)) else: CONSOLE.log("[bold yellow]Warning: could not find existing COLMAP results. Not generating transforms.json") @@ -1030,7 +344,9 @@ class ProcessRecord3D: record3d_image_filenames = list(np.array(record3d_image_filenames)[idx]) # Copy images to output directory - copied_image_paths = copy_images_list(record3d_image_filenames, image_dir=image_dir, verbose=self.verbose) + copied_image_paths = process_data_utils.copy_images_list( + record3d_image_filenames, image_dir=image_dir, verbose=self.verbose + ) num_frames = len(copied_image_paths) copied_image_paths = [Path("images/" + copied_image_path.name) for copied_image_path in copied_image_paths] @@ -1042,10 +358,10 @@ class ProcessRecord3D: ) # Downscale images - summary_log.append(downscale_images(image_dir, self.num_downscales, verbose=self.verbose)) + summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose)) metadata_path = self.data / "metadata.json" - record3d_to_json(copied_image_paths, metadata_path, self.output_dir, indices=idx) + record3d_utils.record3d_to_json(copied_image_paths, metadata_path, self.output_dir, indices=idx) CONSOLE.rule("[bold green]:tada: :tada: :tada: All DONE :tada: :tada: :tada:") for summary in summary_log: @@ -1128,7 +444,7 @@ class ProcessPolycam: polycam_image_filenames = list(np.array(polycam_image_filenames)[idx]) # Copy images to output directory - copied_image_paths = copy_images_list( + copied_image_paths = process_data_utils.copy_images_list( polycam_image_filenames, image_dir=image_dir, crop_border_pixels=self.crop_border_pixels, @@ -1148,14 +464,14 @@ class ProcessPolycam: summary_log.append(f"Started with {num_frames} images") # Downscale images - summary_log.append(downscale_images(image_dir, self.num_downscales, verbose=self.verbose)) + summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose)) # Save json if num_frames == 0: CONSOLE.print("[bold red]No images found, exiting") sys.exit(1) summary_log.extend( - polycam_to_json( + polycam_utils.polycam_to_json( image_filenames=polycam_image_filenames, cameras_dir=polycam_cameras_dir, output_dir=self.output_dir,