diff --git a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py index a712d5005..cffc09be1 100644 --- a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py +++ b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py @@ -289,9 +289,9 @@ class Nerfstudio(DataParser): cameras.rescale_output_resolution(scaling_factor=1.0 / self.downscale_factor) if "applied_transform" in meta: - applied_transform = np.array(meta["applied_transform"], dtype=np.float32) - transform_matrix = transform_matrix @ np.concatenate( - (applied_transform, np.array([0, 0, 0, 1], dtype=np.float32)), 0 + applied_transform = torch.tensor(meta["applied_transform"], dtype=transform_matrix.dtype) + transform_matrix = transform_matrix @ torch.cat( + [applied_transform, torch.tensor([[0, 0, 0, 1]], dtype=transform_matrix.dtype)], 0 ) if "applied_scale" in meta: applied_scale = float(meta["applied_scale"]) diff --git a/nerfstudio/data/utils/colmap_parsing_utils.py b/nerfstudio/data/utils/colmap_parsing_utils.py index 652e2795e..3ca853a27 100644 --- a/nerfstudio/data/utils/colmap_parsing_utils.py +++ b/nerfstudio/data/utils/colmap_parsing_utils.py @@ -4,9 +4,12 @@ This file copied with small modifications from: TODO(1480) Delete this file when moving to pycolmap. -""" -# Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. +""" +# pylint: disable-all +# Disabling all pylinting since this file is copy-pasted over from COLMAP + +# Copyright (c) 2023, ETH Zurich and UNC Chapel Hill. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -35,11 +38,9 @@ TODO(1480) Delete this file when moving to pycolmap. # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # -# Author: Johannes L. Schoenberger (jsch at inf.ethz.ch) - -# pylint: disable-all -# Disabling all pylinting since this file is copy-pasted over from COLMAP +# Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) +import argparse import collections import os import struct @@ -71,6 +72,7 @@ CAMERA_MODELS = { CameraModel(model_id=10, model_name="THIN_PRISM_FISHEYE", num_params=12), } CAMERA_MODEL_IDS = dict([(camera_model.model_id, camera_model) for camera_model in CAMERA_MODELS]) +CAMERA_MODEL_NAMES = dict([(camera_model.model_name, camera_model) for camera_model in CAMERA_MODELS]) def read_next_bytes(fid, num_bytes, format_char_sequence, endian_character="<"): @@ -85,6 +87,22 @@ def read_next_bytes(fid, num_bytes, format_char_sequence, endian_character="<"): return struct.unpack(endian_character + format_char_sequence, data) +def write_next_bytes(fid, data, format_char_sequence, endian_character="<"): + """pack and write to a binary file. + :param fid: + :param data: data to send, if multiple elements are sent at the same time, + they should be encapsuled either in a list or a tuple + :param format_char_sequence: List of {c, e, f, d, h, H, i, I, l, L, q, Q}. + should be the same length as the data list or tuple + :param endian_character: Any of {@, =, <, >, !} + """ + if isinstance(data, (list, tuple)): + bytes = struct.pack(endian_character + format_char_sequence, *data) + else: + bytes = struct.pack(endian_character + format_char_sequence, data) + fid.write(bytes) + + def read_cameras_text(path): """ see: src/base/reconstruction.cc @@ -118,7 +136,7 @@ def read_cameras_binary(path_to_model_file): cameras = {} with open(path_to_model_file, "rb") as fid: num_cameras = read_next_bytes(fid, 8, "Q")[0] - for camera_line_index in range(num_cameras): + for _ in range(num_cameras): camera_properties = read_next_bytes(fid, num_bytes=24, format_char_sequence="iiQQ") camera_id = camera_properties[0] model_id = camera_properties[1] @@ -134,6 +152,42 @@ def read_cameras_binary(path_to_model_file): return cameras +def write_cameras_text(cameras, path): + """ + see: src/base/reconstruction.cc + void Reconstruction::WriteCamerasText(const std::string& path) + void Reconstruction::ReadCamerasText(const std::string& path) + """ + HEADER = ( + "# Camera list with one line of data per camera:\n" + + "# CAMERA_ID, MODEL, WIDTH, HEIGHT, PARAMS[]\n" + + "# Number of cameras: {}\n".format(len(cameras)) + ) + with open(path, "w") as fid: + fid.write(HEADER) + for _, cam in cameras.items(): + to_write = [cam.id, cam.model, cam.width, cam.height, *cam.params] + line = " ".join([str(elem) for elem in to_write]) + fid.write(line + "\n") + + +def write_cameras_binary(cameras, path_to_model_file): + """ + see: src/base/reconstruction.cc + void Reconstruction::WriteCamerasBinary(const std::string& path) + void Reconstruction::ReadCamerasBinary(const std::string& path) + """ + with open(path_to_model_file, "wb") as fid: + write_next_bytes(fid, len(cameras), "Q") + for _, cam in cameras.items(): + model_id = CAMERA_MODEL_NAMES[cam.model].model_id + camera_properties = [cam.id, model_id, cam.width, cam.height] + write_next_bytes(fid, camera_properties, "iiQQ") + for p in cam.params: + write_next_bytes(fid, float(p), "d") + return cameras + + def read_images_text(path): """ see: src/base/reconstruction.cc @@ -178,7 +232,7 @@ def read_images_binary(path_to_model_file): images = {} with open(path_to_model_file, "rb") as fid: num_reg_images = read_next_bytes(fid, 8, "Q")[0] - for image_index in range(num_reg_images): + for _ in range(num_reg_images): binary_image_properties = read_next_bytes(fid, num_bytes=64, format_char_sequence="idddddddi") image_id = binary_image_properties[0] qvec = np.array(binary_image_properties[1:5]) @@ -205,6 +259,57 @@ def read_images_binary(path_to_model_file): return images +def write_images_text(images, path): + """ + see: src/base/reconstruction.cc + void Reconstruction::ReadImagesText(const std::string& path) + void Reconstruction::WriteImagesText(const std::string& path) + """ + if len(images) == 0: + mean_observations = 0 + else: + mean_observations = sum((len(img.point3D_ids) for _, img in images.items())) / len(images) + HEADER = ( + "# Image list with two lines of data per image:\n" + + "# IMAGE_ID, QW, QX, QY, QZ, TX, TY, TZ, CAMERA_ID, NAME\n" + + "# POINTS2D[] as (X, Y, POINT3D_ID)\n" + + "# Number of images: {}, mean observations per image: {}\n".format(len(images), mean_observations) + ) + + with open(path, "w") as fid: + fid.write(HEADER) + for _, img in images.items(): + image_header = [img.id, *img.qvec, *img.tvec, img.camera_id, img.name] + first_line = " ".join(map(str, image_header)) + fid.write(first_line + "\n") + + points_strings = [] + for xy, point3D_id in zip(img.xys, img.point3D_ids): + points_strings.append(" ".join(map(str, [*xy, point3D_id]))) + fid.write(" ".join(points_strings) + "\n") + + +def write_images_binary(images, path_to_model_file): + """ + see: src/base/reconstruction.cc + void Reconstruction::ReadImagesBinary(const std::string& path) + void Reconstruction::WriteImagesBinary(const std::string& path) + """ + with open(path_to_model_file, "wb") as fid: + write_next_bytes(fid, len(images), "Q") + for _, img in images.items(): + write_next_bytes(fid, img.id, "i") + write_next_bytes(fid, img.qvec.tolist(), "dddd") + write_next_bytes(fid, img.tvec.tolist(), "ddd") + write_next_bytes(fid, img.camera_id, "i") + for char in img.name: + write_next_bytes(fid, char.encode("utf-8"), "c") + write_next_bytes(fid, b"\x00", "c") + write_next_bytes(fid, len(img.point3D_ids), "Q") + for xy, p3d_id in zip(img.xys, img.point3D_ids): + write_next_bytes(fid, [*xy, p3d_id], "ddq") + + def read_points3D_text(path): """ see: src/base/reconstruction.cc @@ -232,7 +337,7 @@ def read_points3D_text(path): return points3D -def read_points3d_binary(path_to_model_file): +def read_points3D_binary(path_to_model_file): """ see: src/base/reconstruction.cc void Reconstruction::ReadPoints3DBinary(const std::string& path) @@ -241,7 +346,7 @@ def read_points3d_binary(path_to_model_file): points3D = {} with open(path_to_model_file, "rb") as fid: num_points = read_next_bytes(fid, 8, "Q")[0] - for point_line_index in range(num_points): + for _ in range(num_points): binary_point_line_properties = read_next_bytes(fid, num_bytes=43, format_char_sequence="QdddBBBd") point3D_id = binary_point_line_properties[0] xyz = np.array(binary_point_line_properties[1:4]) @@ -257,7 +362,75 @@ def read_points3d_binary(path_to_model_file): return points3D -def read_model(path, ext): +def write_points3D_text(points3D, path): + """ + see: src/base/reconstruction.cc + void Reconstruction::ReadPoints3DText(const std::string& path) + void Reconstruction::WritePoints3DText(const std::string& path) + """ + if len(points3D) == 0: + mean_track_length = 0 + else: + mean_track_length = sum((len(pt.image_ids) for _, pt in points3D.items())) / len(points3D) + HEADER = ( + "# 3D point list with one line of data per point:\n" + + "# POINT3D_ID, X, Y, Z, R, G, B, ERROR, TRACK[] as (IMAGE_ID, POINT2D_IDX)\n" + + "# Number of points: {}, mean track length: {}\n".format(len(points3D), mean_track_length) + ) + + with open(path, "w") as fid: + fid.write(HEADER) + for _, pt in points3D.items(): + point_header = [pt.id, *pt.xyz, *pt.rgb, pt.error] + fid.write(" ".join(map(str, point_header)) + " ") + track_strings = [] + for image_id, point2D in zip(pt.image_ids, pt.point2D_idxs): + track_strings.append(" ".join(map(str, [image_id, point2D]))) + fid.write(" ".join(track_strings) + "\n") + + +def write_points3D_binary(points3D, path_to_model_file): + """ + see: src/base/reconstruction.cc + void Reconstruction::ReadPoints3DBinary(const std::string& path) + void Reconstruction::WritePoints3DBinary(const std::string& path) + """ + with open(path_to_model_file, "wb") as fid: + write_next_bytes(fid, len(points3D), "Q") + for _, pt in points3D.items(): + write_next_bytes(fid, pt.id, "Q") + write_next_bytes(fid, pt.xyz.tolist(), "ddd") + write_next_bytes(fid, pt.rgb.tolist(), "BBB") + write_next_bytes(fid, pt.error, "d") + track_length = pt.image_ids.shape[0] + write_next_bytes(fid, track_length, "Q") + for image_id, point2D_id in zip(pt.image_ids, pt.point2D_idxs): + write_next_bytes(fid, [image_id, point2D_id], "ii") + + +def detect_model_format(path, ext): + if ( + os.path.isfile(os.path.join(path, "cameras" + ext)) + and os.path.isfile(os.path.join(path, "images" + ext)) + and os.path.isfile(os.path.join(path, "points3D" + ext)) + ): + print("Detected model format: '" + ext + "'") + return True + + return False + + +def read_model(path, ext=""): + # try to detect the extension automatically + if ext == "": + if detect_model_format(path, ".bin"): + ext = ".bin" + elif detect_model_format(path, ".txt"): + ext = ".txt" + else: + print("Provide model format: '.bin' or '.txt'") + return + if ext == ".txt": cameras = read_cameras_text(os.path.join(path, "cameras" + ext)) images = read_images_text(os.path.join(path, "images" + ext)) @@ -265,7 +438,19 @@ def read_model(path, ext): else: cameras = read_cameras_binary(os.path.join(path, "cameras" + ext)) images = read_images_binary(os.path.join(path, "images" + ext)) - points3D = read_points3d_binary(os.path.join(path, "points3D") + ext) + points3D = read_points3D_binary(os.path.join(path, "points3D") + ext) + return cameras, images, points3D + + +def write_model(cameras, images, points3D, path, ext=".bin"): + if ext == ".txt": + write_cameras_text(cameras, os.path.join(path, "cameras" + ext)) + write_images_text(images, os.path.join(path, "images" + ext)) + write_points3D_text(points3D, os.path.join(path, "points3D") + ext) + else: + write_cameras_binary(cameras, os.path.join(path, "cameras" + ext)) + write_images_binary(images, os.path.join(path, "images" + ext)) + write_points3D_binary(points3D, os.path.join(path, "points3D") + ext) return cameras, images, points3D diff --git a/nerfstudio/process_data/colmap_utils.py b/nerfstudio/process_data/colmap_utils.py index 24517a331..859222441 100644 --- a/nerfstudio/process_data/colmap_utils.py +++ b/nerfstudio/process_data/colmap_utils.py @@ -35,7 +35,7 @@ from nerfstudio.data.utils.colmap_parsing_utils import ( qvec2rotmat, read_cameras_binary, read_images_binary, - read_points3d_binary, + read_points3D_binary, ) from nerfstudio.process_data.process_data_utils import CameraModel from nerfstudio.utils import colormaps @@ -458,7 +458,7 @@ def colmap_to_json( applied_transform = np.eye(4)[:3, :] applied_transform = applied_transform[np.array([1, 0, 2]), :] applied_transform[2, :] *= -1 - out["applied_transform"] = applied_transform + out["applied_transform"] = applied_transform.tolist() with open(output_dir / "transforms.json", "w", encoding="utf-8") as f: json.dump(out, f, indent=4) @@ -515,7 +515,7 @@ def create_sfm_depth( # ptid_to_info = recon.points3D # cam_id_to_camera = recon.cameras # im_id_to_image = recon.images - ptid_to_info = read_points3d_binary(recon_dir / "points3D.bin") + ptid_to_info = read_points3D_binary(recon_dir / "points3D.bin") cam_id_to_camera = read_cameras_binary(recon_dir / "cameras.bin") im_id_to_image = read_images_binary(recon_dir / "images.bin") diff --git a/tests/process_data/test_process_images.py b/tests/process_data/test_process_images.py new file mode 100644 index 000000000..3e4fa82a9 --- /dev/null +++ b/tests/process_data/test_process_images.py @@ -0,0 +1,80 @@ +""" +Process images test +""" +import os +from pathlib import Path + +import numpy as np +import torch +from PIL import Image + +from nerfstudio.data.dataparsers.nerfstudio_dataparser import NerfstudioDataParserConfig +from nerfstudio.data.utils.colmap_parsing_utils import Camera +from nerfstudio.data.utils.colmap_parsing_utils import Image as ColmapImage +from nerfstudio.data.utils.colmap_parsing_utils import ( + write_cameras_binary, + write_images_binary, +) +from scripts.process_data import ProcessImages + + +def test_process_images_skip_colmap(tmp_path: Path): + """ + Test ns-process-data images + """ + # Mock a colmap sparse model + width = 100 + height = 150 + sparse_path = tmp_path / "sparse" / "0" + sparse_path.mkdir(exist_ok=True, parents=True) + (tmp_path / "images").mkdir(exist_ok=True, parents=True) + write_cameras_binary( + {1: Camera(1, "OPENCV", width, height, [110, 110, 50, 75, 0, 0, 0, 0, 0, 0])}, + sparse_path / "cameras.bin", + ) + frames = {} + num_frames = 10 + qvecs = np.random.uniform(size=(num_frames, 4)) + tvecs = np.random.uniform(size=(num_frames, 3)) + # original_poses = np.concatenate( + # ( + # np.concatenate( + # ( + # np.stack(list(map(qvec2rotmat, qvecs))), + # tvecs[:, :, None], + # ), + # -1, + # ), + # np.array([[[0, 0, 0, 1]]], dtype=qvecs.dtype).repeat(num_frames, 0), + # ), + # -2, + # ) + for i in range(num_frames): + frames[i + 1] = ColmapImage(i + 1, qvecs[i], tvecs[i], 1, f"image_{i}.png", [], []) + Image.new("RGB", (width, height)).save(tmp_path / "images" / f"image_{i}.png") + write_images_binary(frames, sparse_path / "images.bin") + + # Mock missing COLMAP and ffmpeg in the dev env + old_path = os.environ.get("PATH", "") + os.environ["PATH"] = str(tmp_path / "mocked_bin") + f":{old_path}" + (tmp_path / "mocked_bin").mkdir() + (tmp_path / "mocked_bin" / "colmap").touch(mode=0o777) + (tmp_path / "mocked_bin" / "ffmpeg").touch(mode=0o777) + + # Run ProcessImages + cmd = ProcessImages(tmp_path / "images", tmp_path / "nerfstudio", colmap_model_path=sparse_path, skip_colmap=True) + cmd.main() + os.environ["PATH"] = old_path + + assert (tmp_path / "nerfstudio" / "transforms.json").exists() + parser = NerfstudioDataParserConfig( + data=tmp_path / "nerfstudio", + downscale_factor=None, + orientation_method="none", # orientation_method, + center_method="none", + auto_scale_poses=False, + ).setup() + outputs = parser.get_dataparser_outputs("train") + assert len(outputs.image_filenames) == 9 + assert torch.is_tensor(outputs.dataparser_transform) + # TODO @jkulhanek: Add tests if the loaded poses are the same