Refactor process data (#976)

This commit is contained in:
Matthew Tancik authored and GitHub committed 2022-11-18 09:19:44 -08:00
1 parent 1f85a31120
commit 532f8067fd
10 files changed
+900 -737

No files matched your search

+13
View File
@@ -0,0 +1,13 @@
# Copyright 2022 The Nerfstudio Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+678
View File
@@ -0,0 +1,678 @@
"""
Here we have modified code taken from COLMAP for parsing data in the COLMAP format.
Original file at:
https://github.com/colmap/colmap/blob/1a4d0bad2e90aa65ce997c9d1779518eaed998d5/scripts/python/read_write_model.py.
"""
# Copyright (c) 2022, ETH Zurich and UNC Chapel Hill.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of
# its contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de)
import json
import os
import struct
from dataclasses import dataclass
from io import BufferedReader
from pathlib import Path
from typing import Dict, Optional, Tuple
import appdirs
import numpy as np
import requests
from rich.console import Console
from rich.progress import track
from typing_extensions import Literal
from nerfstudio.process_data.process_data_utils import CameraModel
from nerfstudio.utils.rich_utils import status
from nerfstudio.utils.scripts import run_command
CONSOLE = Console(width=120)
@dataclass
class ColmapCameraModel:
"""Camera model"""
model_id: int
"""Model identifier"""
model_name: str
"""Model name"""
num_params: int
"""Number of parameters"""
@dataclass
class Camera:
"""Camera"""
id: int
"""Camera identifier"""
model: str
"""Camera model"""
width: int
"""Image width"""
height: int
"""Image height"""
params: np.ndarray
"""Camera parameters"""
@dataclass
class Image:
"""Data the corresponds to a single image"""
id: int
"""Image identifier"""
qvec: np.ndarray
"""Quaternion vector"""
tvec: np.ndarray
"""Translation vector"""
camera_id: int
"""Camera identifier"""
name: str
"""Image name"""
xys: np.ndarray
"""2D points"""
point3d_ids: np.ndarray
"""Point3D identifiers"""
@dataclass
class Point3D:
"""Data that corresponds to a single 3D point"""
id: int
"""Point3D identifier"""
xyz: np.ndarray
"""3D point"""
rgb: np.ndarray
"""Color"""
error: float
"""Reconstruction error"""
image_ids: np.ndarray
"""Image identifiers"""
point2d_idxs: np.ndarray
"""Point2D indices"""
COLMAP_CAMERA_MODELS = [
ColmapCameraModel(model_id=0, model_name="SIMPLE_PINHOLE", num_params=3),
ColmapCameraModel(model_id=1, model_name="PINHOLE", num_params=4),
ColmapCameraModel(model_id=2, model_name="SIMPLE_RADIAL", num_params=4),
ColmapCameraModel(model_id=3, model_name="RADIAL", num_params=5),
ColmapCameraModel(model_id=4, model_name="OPENCV", num_params=8),
ColmapCameraModel(model_id=5, model_name="OPENCV_FISHEYE", num_params=8),
ColmapCameraModel(model_id=6, model_name="FULL_OPENCV", num_params=12),
ColmapCameraModel(model_id=7, model_name="FOV", num_params=5),
ColmapCameraModel(model_id=8, model_name="SIMPLE_RADIAL_FISHEYE", num_params=4),
ColmapCameraModel(model_id=9, model_name="RADIAL_FISHEYE", num_params=5),
ColmapCameraModel(model_id=10, model_name="THIN_PRISM_FISHEYE", num_params=12),
]
COLMAP_CAMERA_MODEL_IDS = {camera_model.model_id: camera_model for camera_model in COLMAP_CAMERA_MODELS}
COLMAP_CAMERA_MODEL_NAMES = {camera_model.model_name: camera_model for camera_model in COLMAP_CAMERA_MODELS}
def get_colmap_version(colmap_cmd: str, default_version=3.8) -> float:
"""Returns the version of COLMAP.
This code assumes that colmap returns a version string of the form
"COLMAP 3.8 ..." which may not be true for all versions of COLMAP.
Args:
default_version: Default version to return if COLMAP version can't be determined.
Returns:
The version of COLMAP.
"""
output = run_command(colmap_cmd, verbose=False)
assert output is not None
for line in output.split("\n"):
if line.startswith("COLMAP"):
return float(line.split(" ")[1])
CONSOLE.print(f"[bold red]Could not find COLMAP version. Using default {default_version}")
return default_version
def read_next_bytes(fid: BufferedReader, num_bytes: int, format_char_sequence, endian_character: str = "<"):
"""Read and unpack the next bytes from a binary file.
Args:
fid: Open file
num_bytes: Sum of combination of {2, 4, 8}, e.g. 2, 6, 16, 30, etc.
format_char_sequence: List of {c, e, f, d, h, H, i, I, l, L, q, Q}.
endian_character: Any of {@, =, <, >, !}
Tuple of read and unpacked values.
"""
data = fid.read(num_bytes)
return struct.unpack(endian_character + format_char_sequence, data)
def read_cameras_text(path: Path) -> Dict[int, Camera]:
"""Parse COLMAP cameras.txt file into a dictionary of Camera objects.
Args:
path: Path to cameras.txt file.
Returns:
Dictionary of Camera objects.
"""
cameras = {}
with open(path, encoding="utf-8") as fid:
while True:
line = fid.readline()
if not line:
break
line = line.strip()
if len(line) > 0 and line[0] != "#":
elems = line.split()
camera_id = int(elems[0])
model = elems[1]
width = int(elems[2])
height = int(elems[3])
params = np.array(tuple(map(float, elems[4:])))
cameras[camera_id] = Camera(id=camera_id, model=model, width=width, height=height, params=params)
return cameras
def read_cameras_binary(path_to_model_file: Path) -> Dict[int, Camera]:
"""Parse COLMAP cameras.bin file into a dictionary of Camera objects.
Args:
path_to_model_file: Path to cameras.bin file.
Returns:
Dictionary of Camera objects.
"""
cameras = {}
with open(path_to_model_file, "rb") as fid:
num_cameras = read_next_bytes(fid, 8, "Q")[0]
for _ in range(num_cameras):
camera_properties = read_next_bytes(fid, num_bytes=24, format_char_sequence="iiQQ")
camera_id = camera_properties[0]
model_id = camera_properties[1]
model_name = COLMAP_CAMERA_MODEL_IDS[camera_properties[1]].model_name
width = camera_properties[2]
height = camera_properties[3]
num_params = COLMAP_CAMERA_MODEL_IDS[model_id].num_params
params = read_next_bytes(fid, num_bytes=8 * num_params, format_char_sequence="d" * num_params)
cameras[camera_id] = Camera(
id=camera_id, model=model_name, width=width, height=height, params=np.array(params)
)
assert len(cameras) == num_cameras
return cameras
def read_images_text(path: Path) -> Dict[int, Image]:
"""Parse COLMAP images.txt file into a dictionary of Image objects.
Args:
path: Path to images.txt file.
Returns:
Dictionary of Image objects.
"""
images = {}
with open(path, encoding="utf-8") as fid:
while True:
line = fid.readline()
if not line:
break
line = line.strip()
if len(line) > 0 and line[0] != "#":
elems = line.split()
image_id = int(elems[0])
qvec = np.array(tuple(map(float, elems[1:5])))
tvec = np.array(tuple(map(float, elems[5:8])))
camera_id = int(elems[8])
image_name = elems[9]
elems = fid.readline().split()
xys = np.column_stack([tuple(map(float, elems[0::3])), tuple(map(float, elems[1::3]))])
point3d_ids = np.array(tuple(map(int, elems[2::3])))
images[image_id] = Image(
id=image_id,
qvec=qvec,
tvec=tvec,
camera_id=camera_id,
name=image_name,
xys=xys,
point3d_ids=point3d_ids,
)
return images
def read_images_binary(path_to_model_file: Path) -> Dict[int, Image]:
"""Parse COLMAP images.bin file into a dictionary of Image objects.
Args:
path_to_model_file: Path to images.bin file.
Returns:
Dictionary of Image objects.
"""
images = {}
with open(path_to_model_file, "rb") as fid:
num_reg_images = read_next_bytes(fid, 8, "Q")[0]
for _ in range(num_reg_images):
binary_image_properties = read_next_bytes(fid, num_bytes=64, format_char_sequence="idddddddi")
image_id = binary_image_properties[0]
qvec = np.array(binary_image_properties[1:5])
tvec = np.array(binary_image_properties[5:8])
camera_id = binary_image_properties[8]
image_name = ""
current_char = read_next_bytes(fid, 1, "c")[0]
while current_char != b"\x00": # look for the ASCII 0 entry
image_name += current_char.decode("utf-8")
current_char = read_next_bytes(fid, 1, "c")[0]
num_points2d = read_next_bytes(fid, num_bytes=8, format_char_sequence="Q")[0]
x_y_id_s = read_next_bytes(fid, num_bytes=24 * num_points2d, format_char_sequence="ddq" * num_points2d)
xys = np.column_stack([tuple(map(float, x_y_id_s[0::3])), tuple(map(float, x_y_id_s[1::3]))])
point3d_ids = np.array(tuple(map(int, x_y_id_s[2::3])))
images[image_id] = Image(
id=image_id,
qvec=qvec,
tvec=tvec,
camera_id=camera_id,
name=image_name,
xys=xys,
point3d_ids=point3d_ids,
)
return images
def read_points3d_text(path) -> Dict[int, Point3D]:
"""Parse COLMAP points3D.txt file into a dictionary of Point3D objects.
Args:
path: Path to points3D.txt file.
Returns:
Dictionary of Point3D objects.
"""
points3d = {}
with open(path, encoding="utf-8") as fid:
while True:
line = fid.readline()
if not line:
break
line = line.strip()
if len(line) > 0 and line[0] != "#":
elems = line.split()
point3d_id = int(elems[0])
xyz = np.array(tuple(map(float, elems[1:4])))
rgb = np.array(tuple(map(int, elems[4:7])))
error = float(elems[7])
image_ids = np.array(tuple(map(int, elems[8::2])))
point2d_idxs = np.array(tuple(map(int, elems[9::2])))
points3d[point3d_id] = Point3D(
id=point3d_id, xyz=xyz, rgb=rgb, error=error, image_ids=image_ids, point2d_idxs=point2d_idxs
)
return points3d
def read_points3d_binary(path_to_model_file: Path) -> Dict[int, Point3D]:
"""Parse COLMAP points3D.bin file into a dictionary of Point3D objects.
Args:
path_to_model_file: Path to points3D.bin file.
Returns:
Dictionary of Point3D objects.
"""
points3d = {}
with open(path_to_model_file, "rb") as fid:
num_points = read_next_bytes(fid, 8, "Q")[0]
for _ in range(num_points):
binary_point_line_properties = read_next_bytes(fid, num_bytes=43, format_char_sequence="QdddBBBd")
point3d_id = binary_point_line_properties[0]
xyz = np.array(binary_point_line_properties[1:4])
rgb = np.array(binary_point_line_properties[4:7])
error = np.array(binary_point_line_properties[7])
track_length = read_next_bytes(fid, num_bytes=8, format_char_sequence="Q")[0]
track_elems = read_next_bytes(fid, num_bytes=8 * track_length, format_char_sequence="ii" * track_length)
image_ids = np.array(tuple(map(int, track_elems[0::2])))
point2d_idxs = np.array(tuple(map(int, track_elems[1::2])))
points3d[point3d_id] = Point3D(
id=point3d_id, xyz=xyz, rgb=rgb, error=float(error), image_ids=image_ids, point2d_idxs=point2d_idxs
)
return points3d
def detect_model_format(path: Path, ext: str) -> bool:
"""Detect the format of the model file.
Args:
path: Path to the model file.
ext: Extension to test.
Returns:
True if the model file is the tested extenstion, False otherwise.
"""
if (
os.path.isfile(path / f"cameras{ext}")
and os.path.isfile(path / f"images{ext}")
and os.path.isfile(path / f"points3D{ext}")
):
print("Detected model format: '" + ext + "'")
return True
return False
def read_model(path: Path, ext: Optional[str] = None) -> Tuple[Dict[int, Camera], Dict[int, Image], Dict[int, Point3D]]:
"""Read a COLMAP model from a directory.
Args:
path: Path to the model directory.
ext: Extension of the model files. If None, the function will try to detect the format.
Returns:
Tuple of dictionaries of Camera, Image, and Point3D objects.
"""
# try to detect the extension automatically
if ext is None:
if detect_model_format(path, ".bin"):
ext = ".bin"
elif detect_model_format(path, ".txt"):
ext = ".txt"
else:
raise ValueError("Provide model format: '.bin' or '.txt'")
if ext == ".txt":
cameras = read_cameras_text(path / f"cameras{ext}")
images = read_images_text(path / f"images{ext}")
points3d = read_points3d_text(path / f"points3D{ext}")
else:
cameras = read_cameras_binary(path / f"cameras{ext}")
images = read_images_binary(path / f"images{ext}")
points3d = read_points3d_binary(path / f"points3D{ext}")
return cameras, images, points3d
def qvec2rotmat(qvec) -> np.ndarray:
"""Convert quaternion to rotation matrix.
Args:
qvec: Quaternion vector of shape (4,).
Returns:
Rotation matrix of shape (3, 3).
"""
return np.array(
[
[
1 - 2 * qvec[2] ** 2 - 2 * qvec[3] ** 2,
2 * qvec[1] * qvec[2] - 2 * qvec[0] * qvec[3],
2 * qvec[3] * qvec[1] + 2 * qvec[0] * qvec[2],
],
[
2 * qvec[1] * qvec[2] + 2 * qvec[0] * qvec[3],
1 - 2 * qvec[1] ** 2 - 2 * qvec[3] ** 2,
2 * qvec[2] * qvec[3] - 2 * qvec[0] * qvec[1],
],
[
2 * qvec[3] * qvec[1] - 2 * qvec[0] * qvec[2],
2 * qvec[2] * qvec[3] + 2 * qvec[0] * qvec[1],
1 - 2 * qvec[1] ** 2 - 2 * qvec[2] ** 2,
],
]
)
def rotmat2qvec(R):
"""Convert rotation matrix to quaternion.
Args:
R: Rotation matrix of shape (3, 3).
Returns:
Quaternion vector of shape (4,).
"""
rxx, ryx, rzx, rxy, ryy, rzy, rxz, ryz, rzz = R.flat
K = (
np.array(
[
[rxx - ryy - rzz, 0, 0, 0],
[ryx + rxy, ryy - rxx - rzz, 0, 0],
[rzx + rxz, rzy + ryz, rzz - rxx - ryy, 0],
[ryz - rzy, rzx - rxz, rxy - ryx, rxx + ryy + rzz],
]
)
/ 3.0
)
eigvals, eigvecs = np.linalg.eigh(K)
qvec = eigvecs[np.array([3, 0, 1, 2]), np.argmax(eigvals)]
if qvec[0] < 0:
qvec *= -1
return qvec
def get_vocab_tree() -> Path:
"""Return path to vocab tree. Downloads vocab tree if it doesn't exist.
Returns:
The path to the vocab tree.
"""
vocab_tree_filename = Path(appdirs.user_data_dir("nerfstudio")) / "vocab_tree.fbow"
if not vocab_tree_filename.exists():
r = requests.get("https://demuc.de/colmap/vocab_tree_flickr100K_words32K.bin", stream=True)
vocab_tree_filename.parent.mkdir(parents=True, exist_ok=True)
with open(vocab_tree_filename, "wb") as f:
total_length = r.headers.get("content-length")
assert total_length is not None
for chunk in track(
r.iter_content(chunk_size=1024),
total=int(total_length) / 1024 + 1,
description="Downloading vocab tree...",
):
if chunk:
f.write(chunk)
f.flush()
return vocab_tree_filename
def run_colmap(
image_dir: Path,
colmap_dir: Path,
camera_model: CameraModel,
gpu: bool = True,
verbose: bool = False,
matching_method: Literal["vocab_tree", "exhaustive", "sequential"] = "vocab_tree",
colmap_cmd: str = "colmap",
) -> None:
"""Runs COLMAP on the images.
Args:
image_dir: Path to the directory containing the images.
colmap_dir: Path to the output directory.
camera_model: Camera model to use.
gpu: If True, use GPU.
verbose: If True, logs the output of the command.
"""
colmap_version = get_colmap_version(colmap_cmd)
colmap_database_path = colmap_dir / "database.db"
if colmap_database_path.exists():
# Can't use missing_ok argument because of Python 3.7 compatibility.
colmap_database_path.unlink()
# Feature extraction
feature_extractor_cmd = [
f"{colmap_cmd} feature_extractor",
f"--database_path {colmap_dir / 'database.db'}",
f"--image_path {image_dir}",
"--ImageReader.single_camera 1",
f"--ImageReader.camera_model {camera_model.value}",
f"--SiftExtraction.use_gpu {int(gpu)}",
]
feature_extractor_cmd = " ".join(feature_extractor_cmd)
with status(msg="[bold yellow]Running COLMAP feature extractor...", spinner="moon", verbose=verbose):
run_command(feature_extractor_cmd, verbose=verbose)
CONSOLE.log("[bold green]:tada: Done extracting COLMAP features.")
# Feature matching
feature_matcher_cmd = [
f"{colmap_cmd} {matching_method}_matcher",
f"--database_path {colmap_dir / 'database.db'}",
f"--SiftMatching.use_gpu {int(gpu)}",
]
if matching_method == "vocab_tree":
vocab_tree_filename = get_vocab_tree()
feature_matcher_cmd.append(f"--VocabTreeMatching.vocab_tree_path {vocab_tree_filename}")
feature_matcher_cmd = " ".join(feature_matcher_cmd)
with status(msg="[bold yellow]Running COLMAP feature matcher...", spinner="runner", verbose=verbose):
run_command(feature_matcher_cmd, verbose=verbose)
CONSOLE.log("[bold green]:tada: Done matching COLMAP features.")
# Bundle adjustment
sparse_dir = colmap_dir / "sparse"
sparse_dir.mkdir(parents=True, exist_ok=True)
mapper_cmd = [
f"{colmap_cmd} mapper",
f"--database_path {colmap_dir / 'database.db'}",
f"--image_path {image_dir}",
f"--output_path {sparse_dir}",
]
if colmap_version >= 3.7:
mapper_cmd.append("--Mapper.ba_global_function_tolerance 1e-6")
mapper_cmd = " ".join(mapper_cmd)
with status(
msg="[bold yellow]Running COLMAP bundle adjustment... (This may take a while)",
spinner="circle",
verbose=verbose,
):
run_command(mapper_cmd, verbose=verbose)
CONSOLE.log("[bold green]:tada: Done COLMAP bundle adjustment.")
with status(msg="[bold yellow]Refine intrinsics...", spinner="dqpb", verbose=verbose):
bundle_adjuster_cmd = [
f"{colmap_cmd} bundle_adjuster",
f"--input_path {sparse_dir}/0",
f"--output_path {sparse_dir}/0",
"--BundleAdjustment.refine_principal_point 1",
]
run_command(" ".join(bundle_adjuster_cmd), verbose=verbose)
CONSOLE.log("[bold green]:tada: Done refining intrinsics.")
def colmap_to_json(cameras_path: Path, images_path: Path, output_dir: Path, camera_model: CameraModel) -> int:
"""Converts COLMAP's cameras.bin and images.bin to a JSON file.
Args:
cameras_path: Path to the cameras.bin file.
images_path: Path to the images.bin file.
output_dir: Path to the output directory.
camera_model: Camera model used.
Returns:
The number of registered images.
"""
cameras = read_cameras_binary(cameras_path)
images = read_images_binary(images_path)
# Only supports one camera
camera_params = cameras[1].params
frames = []
for _, im_data in images.items():
rotation = qvec2rotmat(im_data.qvec)
translation = im_data.tvec.reshape(3, 1)
w2c = np.concatenate([rotation, translation], 1)
w2c = np.concatenate([w2c, np.array([[0, 0, 0, 1]])], 0)
c2w = np.linalg.inv(w2c)
# Convert from COLMAP's camera coordinate system to ours
c2w[0:3, 1:3] *= -1
c2w = c2w[np.array([1, 0, 2, 3]), :]
c2w[2, :] *= -1
name = Path(f"./images/{im_data.name}")
frame = {
"file_path": name.as_posix(),
"transform_matrix": c2w.tolist(),
}
frames.append(frame)
out = {
"fl_x": float(camera_params[0]),
"fl_y": float(camera_params[1]),
"cx": float(camera_params[2]),
"cy": float(camera_params[3]),
"w": cameras[1].width,
"h": cameras[1].height,
"camera_model": camera_model.value,
}
if camera_model == CameraModel.OPENCV:
out.update(
{
"k1": float(camera_params[4]),
"k2": float(camera_params[5]),
"p1": float(camera_params[6]),
"p2": float(camera_params[7]),
}
)
if camera_model == CameraModel.OPENCV_FISHEYE:
out.update(
{
"k1": float(camera_params[4]),
"k2": float(camera_params[5]),
"k3": float(camera_params[6]),
"k4": float(camera_params[7]),
}
)
out["frames"] = frames
with open(output_dir / "transforms.json", "w", encoding="utf-8") as f:
json.dump(out, f, indent=4)
return len(frames)
def get_matching_summary(num_intial_frames: int, num_matched_frames: int) -> str:
"""Returns a summary of the matching results.
Args:
num_intial_frames: The number of initial frames.
num_matched_frames: The number of matched frames.
Returns:
A summary of the matching results.
"""
match_ratio = num_matched_frames / num_intial_frames
if match_ratio == 1:
return "[bold green]COLAMP found poses for all images, CONGRATS!"
if match_ratio < 0.4:
result = f"[bold red]COLMAP only found poses for {num_matched_frames / num_intial_frames * 100:.2f}%"
result += " of the images. This is low.\nThis can be caused by a variety of reasons,"
result += " such poor scene coverage, blurry images, or large exposure changes."
return result
if match_ratio < 0.8:
result = f"[bold yellow]COLMAP only found poses for {num_matched_frames / num_intial_frames * 100:.2f}%"
result += " of the images.\nThis isn't great, but may be ok."
result += "\nMissing poses can be caused by a variety of reasons, such poor scene coverage, blurry images,"
result += " or large exposure changes."
return result
return f"[bold green]COLMAP found poses for {num_matched_frames / num_intial_frames * 100:.2f}% of the images."
+125
View File
@@ -0,0 +1,125 @@
# Copyright 2022 The Nerfstudio Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Helper utils for processing insta360 data."""
import sys
from pathlib import Path
from typing import List, Tuple
from rich.console import Console
from nerfstudio.process_data.process_data_utils import get_num_frames_in_video
from nerfstudio.utils.rich_utils import status
from nerfstudio.utils.scripts import run_command
CONSOLE = Console(width=120)
def get_insta360_filenames(data: Path) -> Tuple[Path, Path]:
"""Returns the filenames of the Insta360 videos from a single video file.
Example input name: VID_20220212_070353_00_003.insv
Args:
data: Path to a Insta360 file.
Returns:
The filenames of the Insta360 videios.
"""
if data.suffix != ".insv":
raise ValueError("The input file must be an .insv file.")
file_parts = data.stem.split("_")
stem_back = f"VID_{file_parts[1]}_{file_parts[2]}_00_{file_parts[4]}.insv"
stem_front = f"VID_{file_parts[1]}_{file_parts[2]}_10_{file_parts[4]}.insv"
filename_back = data.parent / stem_back
filename_front = data.parent / stem_front
if not filename_back.exists():
raise FileNotFoundError(f"Could not find {filename_back}")
if not filename_front.exists():
raise FileNotFoundError(f"Could not find {filename_front}")
return filename_back, filename_front
def convert_insta360_to_images(
video_front: Path,
video_back: Path,
image_dir: Path,
num_frames_target: int,
crop_percentage: float = 0.7,
verbose: bool = False,
) -> Tuple[List[str], int]:
"""Converts a video into a sequence of images.
Args:
video_front: Path to the front video.
video_back: Path to the back video.
output_dir: Path to the output directory.
num_frames_target: Number of frames to extract.
verbose: If True, logs the output of the command.
Returns:
A tuple containing summary of the conversion and the number of extracted frames.
"""
with status(msg="Converting video to images...", spinner="bouncingBall", verbose=verbose):
# delete existing images in folder
for img in image_dir.glob("*.png"):
if verbose:
CONSOLE.log(f"Deleting {img}")
img.unlink()
num_frames_front = get_num_frames_in_video(video_front)
num_frames_back = get_num_frames_in_video(video_back)
if num_frames_front == 0:
CONSOLE.print(f"[bold red]Error: Video has no frames: {video_front}")
sys.exit(1)
if num_frames_back == 0:
CONSOLE.print(f"[bold red]Error: Video has no frames: {video_front}")
sys.exit(1)
spacing = num_frames_front // (num_frames_target // 2)
vf_cmds = []
if spacing > 1:
vf_cmds = [f"thumbnail={spacing}", "setpts=N/TB"]
else:
CONSOLE.print("[bold red]Can't satify requested number of frames. Extracting all frames.")
vf_cmds.append(f"crop=iw*{crop_percentage}:ih*{crop_percentage}")
front_vf_cmds = vf_cmds + ["transpose=2"]
back_vf_cmds = vf_cmds + ["transpose=1"]
front_ffmpeg_cmd = f"ffmpeg -i {video_front} -vf {','.join(front_vf_cmds)} -r 1 {image_dir / 'frame_%05d.png'}"
back_ffmpeg_cmd = (
f"ffmpeg -i {video_back} -vf {','.join(back_vf_cmds)} -r 1 {image_dir / 'back_frame_%05d.png'}"
)
run_command(front_ffmpeg_cmd, verbose=verbose)
run_command(back_ffmpeg_cmd, verbose=verbose)
num_extracted_front_frames = len(list(image_dir.glob("frame*.png")))
for i, img in enumerate(image_dir.glob("back_frame_*.png")):
img.rename(image_dir / f"frame_{i+1+num_extracted_front_frames:05d}.png")
num_final_frames = len(list(image_dir.glob("*.png")))
summary_log = []
summary_log.append(f"Starting with {num_frames_front + num_frames_back} video frames")
summary_log.append(f"We extracted {num_final_frames} images")
CONSOLE.log("[bold green]:tada: Done converting insta360 to images.")
return summary_log, num_final_frames
+97
View File
@@ -0,0 +1,97 @@
# Copyright 2022 The Nerfstudio Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Helper utils for processing polycam data into the nerfstudio format."""
import json
import sys
from pathlib import Path
from typing import List
from rich.console import Console
from nerfstudio.process_data.process_data_utils import CAMERA_MODELS
from nerfstudio.utils import io
CONSOLE = Console(width=120)
def polycam_to_json(
image_filenames: List[Path],
cameras_dir: Path,
output_dir: Path,
min_blur_score: float = 0.0,
crop_border_pixels: int = 0,
) -> List[str]:
"""Convert Polycam data into a nerfstudio dataset.
Args:
image_filenames: List of paths to the original images.
cameras_dir: Path to the polycam cameras directory.
output_dir: Path to the output directory.
min_blur_score: Minimum blur score to use an image. Images below this value will be skipped.
crop_border_pixels: Number of pixels to crop from each border of the image.
Returns:
Summary of the conversion.
"""
frame_json = io.load_from_json(cameras_dir / f"{image_filenames[0].stem}.json")
# TODO Add per-frame intrinsics
data = {}
data["fl_x"] = frame_json["fx"]
data["fl_y"] = frame_json["fy"]
data["cx"] = frame_json["cx"] - crop_border_pixels
data["cy"] = frame_json["cy"] - crop_border_pixels
data["w"] = frame_json["width"] - crop_border_pixels * 2
data["h"] = frame_json["height"] - crop_border_pixels * 2
data["camera_model"] = CAMERA_MODELS["perspective"].value
# Needs to be a string for camera_utils.auto_orient_and_center_poses
data["orientation_override"] = "none"
frames = []
skipped_frames = 0
for i, image_filename in enumerate(image_filenames):
json_filename = cameras_dir / f"{image_filename.stem}.json"
frame_json = io.load_from_json(json_filename)
if "blur_score" in frame_json and frame_json["blur_score"] < min_blur_score:
skipped_frames += 1
continue
frame = {}
frame["file_path"] = f"./images/frame_{i+1:05d}{image_filename.suffix}"
# Transform matrix to nerfstudio format. Please refer to the documentation for coordinate system conventions.
frame["transform_matrix"] = [
[frame_json["t_20"], frame_json["t_21"], frame_json["t_22"], frame_json["t_23"]],
[frame_json["t_00"], frame_json["t_01"], frame_json["t_02"], frame_json["t_03"]],
[frame_json["t_10"], frame_json["t_11"], frame_json["t_12"], frame_json["t_13"]],
[0.0, 0.0, 0.0, 1.0],
]
frames.append(frame)
data["frames"] = frames
with open(output_dir / "transforms.json", "w", encoding="utf-8") as f:
json.dump(data, f, indent=4)
summary = []
if skipped_frames > 0:
summary.append(f"Skipped {skipped_frames} frames due to low blur score.")
summary.append(f"Final dataset is {len(image_filenames) - skipped_frames} frames.")
if len(image_filenames) - skipped_frames == 0:
CONSOLE.print("[bold red]No images remain after filtering, exiting")
sys.exit(1)
return summary
@@ -0,0 +1,211 @@
# Copyright 2022 The Nerfstudio Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Helper utils for processing data into the nerfstudio format."""
import shutil
import sys
from enum import Enum
from pathlib import Path
from typing import List, Optional, Tuple
from rich.console import Console
from nerfstudio.utils.rich_utils import status
from nerfstudio.utils.scripts import run_command
CONSOLE = Console(width=120)
class CameraModel(Enum):
"""Enum for camera types."""
OPENCV = "OPENCV"
OPENCV_FISHEYE = "OPENCV_FISHEYE"
CAMERA_MODELS = {
"perspective": CameraModel.OPENCV,
"fisheye": CameraModel.OPENCV_FISHEYE,
}
def get_num_frames_in_video(video: Path) -> int:
"""Returns the number of frames in a video.
Args:
video: Path to a video.
Returns:
The number of frames in a video.
"""
cmd = f"ffprobe -v error -select_streams v:0 -count_packets \
-show_entries stream=nb_read_packets -of csv=p=0 {video}"
output = run_command(cmd)
assert output is not None
output = output.strip(" ,\t\n\r")
return int(output)
def convert_video_to_images(
video_path: Path, image_dir: Path, num_frames_target: int, verbose: bool = False
) -> Tuple[List[str], int]:
"""Converts a video into a sequence of images.
Args:
video_path: Path to the video.
output_dir: Path to the output directory.
num_frames_target: Number of frames to extract.
verbose: If True, logs the output of the command.
Returns:
A tuple containing summary of the conversion and the number of extracted frames.
"""
with status(msg="Converting video to images...", spinner="bouncingBall", verbose=verbose):
# delete existing images in folder
for img in image_dir.glob("*.png"):
if verbose:
CONSOLE.log(f"Deleting {img}")
img.unlink()
num_frames = get_num_frames_in_video(video_path)
if num_frames == 0:
CONSOLE.print(f"[bold red]Error: Video has no frames: {video_path}")
sys.exit(1)
print("Number of frames in video:", num_frames)
out_filename = image_dir / "frame_%05d.png"
ffmpeg_cmd = f"ffmpeg -i {video_path}"
spacing = num_frames // num_frames_target
if spacing > 1:
ffmpeg_cmd += f" -vf thumbnail={spacing},setpts=N/TB -r 1"
else:
CONSOLE.print("[bold red]Can't satify requested number of frames. Extracting all frames.")
ffmpeg_cmd += f" {out_filename}"
run_command(ffmpeg_cmd, verbose=verbose)
num_final_frames = len(list(image_dir.glob("*.png")))
summary_log = []
summary_log.append(f"Starting with {num_frames} video frames")
summary_log.append(f"We extracted {num_final_frames} images")
CONSOLE.log("[bold green]:tada: Done converting video to images.")
return summary_log, num_final_frames
def copy_images_list(
image_paths: List[Path], image_dir: Path, crop_border_pixels: Optional[int] = None, verbose: bool = False
) -> List[Path]:
"""Copy all images in a list of Paths. Useful for filtering from a directory.
Args:
image_paths: List of Paths of images to copy to a new directory.
image_dir: Path to the output directory.
crop_border_pixels: If not None, crops each edge by the specified number of pixels.
verbose: If True, print extra logging.
Returns:
A list of the copied image Paths.
"""
# Remove original directory only if we provide a proper image folder path
if image_dir.is_dir() and len(image_paths):
shutil.rmtree(image_dir, ignore_errors=True)
image_dir.mkdir(exist_ok=True, parents=True)
copied_image_paths = []
# Images should be 1-indexed for the rest of the pipeline.
for idx, image_path in enumerate(image_paths):
if verbose:
CONSOLE.log(f"Copying image {idx + 1} of {len(image_paths)}...")
copied_image_path = image_dir / f"frame_{idx + 1:05d}{image_path.suffix}"
shutil.copy(image_path, copied_image_path)
copied_image_paths.append(copied_image_path)
if crop_border_pixels is not None:
file_type = image_paths[0].suffix
filename = f"frame_%05d{file_type}"
crop = f"crop=iw-{crop_border_pixels*2}:ih-{crop_border_pixels*2}"
ffmpeg_cmd = f"ffmpeg -y -i {image_dir / filename} -q:v 2 -vf {crop} {image_dir / filename}"
run_command(ffmpeg_cmd, verbose=verbose)
num_frames = len(image_paths)
if num_frames == 0:
CONSOLE.log("[bold red]:skull: No usable images in the data folder.")
else:
CONSOLE.log("[bold green]:tada: Done copying images.")
return copied_image_paths
def copy_images(data: Path, image_dir: Path, verbose) -> int:
"""Copy images from a directory to a new directory.
Args:
data: Path to the directory of images.
image_dir: Path to the output directory.
verbose: If True, print extra logging.
Returns:
The number of images copied.
"""
with status(msg="[bold yellow]Copying images...", spinner="bouncingBall", verbose=verbose):
allowed_exts = [".jpg", ".jpeg", ".png", ".tif", ".tiff"]
image_paths = sorted([p for p in data.glob("[!.]*") if p.suffix.lower() in allowed_exts])
num_frames = len(copy_images_list(image_paths, image_dir, verbose))
return num_frames
def downscale_images(image_dir: Path, num_downscales: int, verbose: bool = False) -> str:
"""Downscales the images in the directory. Uses FFMPEG.
Assumes images are named frame_00001.png, frame_00002.png, etc.
Args:
image_dir: Path to the directory containing the images.
num_downscales: Number of times to downscale the images. Downscales by 2 each time.
verbose: If True, logs the output of the command.
Returns:
Summary of downscaling.
"""
if num_downscales == 0:
return "No downscaling performed."
with status(msg="[bold yellow]Downscaling images...", spinner="growVertical", verbose=verbose):
downscale_factors = [2**i for i in range(num_downscales + 1)[1:]]
for downscale_factor in downscale_factors:
assert downscale_factor > 1
assert isinstance(downscale_factor, int)
downscale_dir = image_dir.parent / f"images_{downscale_factor}"
downscale_dir.mkdir(parents=True, exist_ok=True)
file_type = image_dir.glob("frame_*").__next__().suffix
filename = f"frame_%05d{file_type}"
ffmpeg_cmd = [
f"ffmpeg -i {image_dir / filename} ",
f"-q:v 2 -vf scale=iw/{downscale_factor}:ih/{downscale_factor} ",
f"{downscale_dir / filename}",
]
ffmpeg_cmd = " ".join(ffmpeg_cmd)
run_command(ffmpeg_cmd, verbose=verbose)
CONSOLE.log("[bold green]:tada: Done downscaling images.")
downscale_text = [f"[bold blue]{2**(i+1)}x[/bold blue]" for i in range(num_downscales)]
downscale_text = ", ".join(downscale_text[:-1]) + " and " + downscale_text[-1]
return f"We downsampled the images by {downscale_text}"
+94
View File
@@ -0,0 +1,94 @@
# Copyright 2022 The Nerfstudio Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Helper functions for processing record3d data."""
import json
from pathlib import Path
from typing import List
import numpy as np
from rich.console import Console
from scipy.spatial.transform import Rotation
from nerfstudio.process_data.process_data_utils import CAMERA_MODELS
from nerfstudio.utils import io
CONSOLE = Console(width=120)
def record3d_to_json(images_paths: List[Path], metadata_path: Path, output_dir: Path, indices: np.ndarray) -> int:
"""Converts Record3D's metadata and image paths to a JSON file.
Args:
images_paths: list if image paths.
metadata_path: Path to the Record3D metadata JSON file.
output_dir: Path to the output directory.
indices: Indices to sample the metadata_path. Should be the same length as images_paths.
Returns:
The number of registered images.
"""
assert len(images_paths) == len(indices)
metadata_dict = io.load_from_json(metadata_path)
poses_data = np.array(metadata_dict["poses"]) # (N, 3, 4)
camera_to_worlds = np.concatenate(
[Rotation.from_quat(poses_data[:, :4]).as_matrix(), poses_data[:, 4:, None]],
axis=-1,
).astype(np.float32)
camera_to_worlds = camera_to_worlds[indices]
homogeneous_coord = np.zeros_like(camera_to_worlds[..., :1, :])
homogeneous_coord[..., :, 3] = 1
camera_to_worlds = np.concatenate([camera_to_worlds, homogeneous_coord], -2)
frames = []
for i, im_path in enumerate(images_paths):
c2w = camera_to_worlds[i]
frame = {
"file_path": im_path.as_posix(),
"transform_matrix": c2w.tolist(),
}
frames.append(frame)
# Camera intrinsics
K = np.array(metadata_dict["K"]).reshape((3, 3)).T
focal_length = K[0, 0]
H = metadata_dict["h"]
W = metadata_dict["w"]
# TODO(akristoffersen): The metadata dict comes with principle points,
# but caused errors in image coord indexing. Should update once that is fixed.
cx, cy = W / 2, H / 2
out = {
"fl_x": focal_length,
"fl_y": focal_length,
"cx": cx,
"cy": cy,
"w": W,
"h": H,
"camera_model": CAMERA_MODELS["perspective"].name,
}
out["frames"] = frames
with open(output_dir / "transforms.json", "w", encoding="utf-8") as f:
json.dump(out, f, indent=4)
return len(frames)