diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..e1590ed27 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,190 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +!scripts/downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Experiments and outputs +outputs/ +exports/ +renders/ +# tensorboard log files +events.out.* + +# Data +data + +# Misc +old/ +temp* +!temporal* +.nfs* +external/ +__MACOSX/ +node_modules/ +bash/ +cache/ +package-lock.json +camera_paths/ +.DS_Store +._.DS_Store +*/**/.DS_Store +*/**/._.DS_Store + diff --git a/.gitignore b/.gitignore index d6f0cf5d9..ab8a4e108 100644 --- a/.gitignore +++ b/.gitignore @@ -184,3 +184,8 @@ bash/ cache/ package-lock.json camera_paths/ +.DS_Store +._.DS_Store +*/**/.DS_Store +*/**/._.DS_Store + diff --git a/Dockerfile b/Dockerfile index 26f6541e1..2bdb5328b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,10 +48,12 @@ RUN apt-get update && \ libsuitesparse-dev \ nano \ protobuf-compiler \ + python-is-python3 \ python3.10-dev \ python3-pip \ qtbase5-dev \ sudo \ + vim-tiny \ wget && \ rm -rf /var/lib/apt/lists/* @@ -62,7 +64,7 @@ RUN git clone --branch v0.6.0 https://github.com/google/glog.git --single-branch mkdir build && \ cd build && \ cmake .. && \ - make -j && \ + make -j `nproc` && \ make install && \ cd ../.. && \ rm -rf glog @@ -76,7 +78,7 @@ RUN git clone --branch 2.1.0 https://ceres-solver.googlesource.com/ceres-solver. mkdir build && \ cd build && \ cmake .. -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF && \ - make -j && \ + make -j `nproc` && \ make install && \ cd ../.. && \ rm -rf ceres-solver @@ -87,13 +89,12 @@ RUN git clone --branch 3.8 https://github.com/colmap/colmap.git --single-branch mkdir build && \ cd build && \ cmake .. -DCUDA_ENABLED=ON \ - -DCUDA_NVCC_FLAGS="--std c++14" \ -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES} && \ - make -j && \ + make -j `nproc` && \ make install && \ cd ../.. && \ rm -rf colmap - + # Create non root user and setup environment. RUN useradd -m -d /home/user -g root -G sudo -u 1000 user RUN usermod -aG sudo user @@ -112,13 +113,14 @@ SHELL ["/bin/bash", "-c"] # Upgrade pip and install packages. RUN python3.10 -m pip install --upgrade pip setuptools pathtools promise pybind11 -# Install pytorch and submodules (Currently, we still use cu116 which is the latest version for toch 1.12.1 and is compatible with CUDA 11.8). +# Install pytorch and submodules (Currently, we still use cu116 which is the latest version for torch 1.12.1 and is compatible with CUDA 11.8). RUN python3.10 -m pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116 # Install tynyCUDNN (we need to set the target architectures as environment variable first). ENV TCNN_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES} -RUN python3.10 -m pip install git+https://github.com/NVlabs/tiny-cuda-nn.git#subdirectory=bindings/torch +RUN python3.10 -m pip install git+https://github.com/NVlabs/tiny-cuda-nn.git@v1.6#subdirectory=bindings/torch # Install pycolmap 0.3.0, required by hloc. +# TODO(https://github.com/colmap/pycolmap/issues/111) use wheel when available for Python 3.10 RUN git clone --branch v0.3.0 --recursive https://github.com/colmap/pycolmap.git && \ cd pycolmap && \ python3.10 -m pip install . && \ diff --git a/docs/quickstart/installation.md b/docs/quickstart/installation.md index 560ec9019..2a2db4453 100644 --- a/docs/quickstart/installation.md +++ b/docs/quickstart/installation.md @@ -135,6 +135,14 @@ or be built from the repository using ```bash docker build --tag nerfstudio -f Dockerfile . ``` + +To restrict to only CUDA architectures that you have available locally, use the `CUDA_ARCHITECTURES` +build arg and look up [the compute capability for your GPU](https://developer.nvidia.com/cuda-gpus). +For example, here's how to build with support for GeForce 30xx series GPUs: +```bash +docker build --build-arg CUDA_ARCHITECTURES=86 -tag nerfstudio-86 . +``` + ### Using an interactive container The docker container can be launched with an interactive terminal where nerfstudio commands can be entered as usual. Some parameters are required and some are strongly recommended for usage as following: ```bash diff --git a/nerfstudio/cameras/cameras.py b/nerfstudio/cameras/cameras.py index 80b7f6487..01b71bfdb 100644 --- a/nerfstudio/cameras/cameras.py +++ b/nerfstudio/cameras/cameras.py @@ -19,7 +19,6 @@ import base64 import math import os from dataclasses import dataclass -from distutils.util import strtobool from enum import Enum, auto from typing import Dict, List, Optional, Tuple, Union @@ -34,6 +33,7 @@ import nerfstudio.utils.poses as pose_utils from nerfstudio.cameras import camera_utils from nerfstudio.cameras.rays import RayBundle from nerfstudio.data.scene_box import SceneBox +from nerfstudio.utils.misc import strtobool from nerfstudio.utils.tensor_dataclass import TensorDataclass diff --git a/nerfstudio/data/dataparsers/nuscenes_dataparser.py b/nerfstudio/data/dataparsers/nuscenes_dataparser.py index e74b59acd..6b35cf356 100644 --- a/nerfstudio/data/dataparsers/nuscenes_dataparser.py +++ b/nerfstudio/data/dataparsers/nuscenes_dataparser.py @@ -20,6 +20,7 @@ from pathlib import Path from typing import Optional, Tuple, Type import numpy as np +import pyquaternion import torch from nuscenes.nuscenes import NuScenes as NuScenesDatabase from typing_extensions import Literal @@ -31,14 +32,19 @@ from nerfstudio.data.dataparsers.base_dataparser import ( DataparserOutputs, ) from nerfstudio.data.scene_box import SceneBox -from nerfstudio.process_data.colmap_utils import qvec2rotmat -def rotation_translation_to_pose(r_vec, t_vec): +def rotation_translation_to_pose(r_quat, t_vec): """Convert quaternion rotation and translation vectors to 4x4 matrix""" pose = np.eye(4) - pose[:3, :3] = qvec2rotmat(r_vec) + + # NB: Nuscenes recommends pyquaternion, which uses scalar-first format (w x y z) + # https://github.com/nutonomy/nuscenes-devkit/issues/545#issuecomment-766509242 + # https://github.com/KieranWynn/pyquaternion/blob/99025c17bab1c55265d61add13375433b35251af/pyquaternion/quaternion.py#L299 + # https://fzheng.me/2017/11/12/quaternion_conventions_en/ + pose[:3, :3] = pyquaternion.Quaternion(r_quat).rotation_matrix + pose[:3, 3] = t_vec return pose diff --git a/nerfstudio/data/dataparsers/phototourism_dataparser.py b/nerfstudio/data/dataparsers/phototourism_dataparser.py index 5af4518fc..5b5287d3d 100644 --- a/nerfstudio/data/dataparsers/phototourism_dataparser.py +++ b/nerfstudio/data/dataparsers/phototourism_dataparser.py @@ -33,7 +33,13 @@ from nerfstudio.data.dataparsers.base_dataparser import ( DataparserOutputs, ) from nerfstudio.data.scene_box import SceneBox -from nerfstudio.data.utils.colmap_utils import read_cameras_binary, read_images_binary + +# TODO(1480) use pycolmap instead of colmap_parsing_utils +# import pycolmap +from nerfstudio.data.utils.colmap_parsing_utils import ( + read_cameras_binary, + read_images_binary, +) CONSOLE = Console(width=120) @@ -76,11 +82,14 @@ class Phototourism(DataParser): # pylint: disable=too-many-statements def _generate_dataparser_outputs(self, split="train"): - image_filenames = [] poses = [] with CONSOLE.status(f"[bold green]Reading phototourism images and poses for {split} split...") as _: + # TODO(1480) use pycolmap + # recon = pycolmap.Reconstruction(self.data / "dense" / "sparse") + # cams = recon.cameras + # imgs = recon.images cams = read_cameras_binary(self.data / "dense/sparse/cameras.bin") imgs = read_images_binary(self.data / "dense/sparse/images.bin") diff --git a/nerfstudio/data/utils/colmap_utils.py b/nerfstudio/data/utils/colmap_parsing_utils.py similarity index 98% rename from nerfstudio/data/utils/colmap_utils.py rename to nerfstudio/data/utils/colmap_parsing_utils.py index 2615d00d9..652e2795e 100644 --- a/nerfstudio/data/utils/colmap_utils.py +++ b/nerfstudio/data/utils/colmap_parsing_utils.py @@ -1,3 +1,11 @@ +""" +This file copied with small modifications from: + * https://github.com/colmap/colmap/blob/1a4d0bad2e90aa65ce997c9d1779518eaed998d5/scripts/python/read_write_model.py + +TODO(1480) Delete this file when moving to pycolmap. + +""" + # Copyright (c) 2018, ETH Zurich and UNC Chapel Hill. # All rights reserved. # diff --git a/nerfstudio/process_data/colmap_utils.py b/nerfstudio/process_data/colmap_utils.py index f9613dbc1..064c2110a 100644 --- a/nerfstudio/process_data/colmap_utils.py +++ b/nerfstudio/process_data/colmap_utils.py @@ -1,145 +1,50 @@ -""" -Here we have modified code taken from COLMAP for parsing data in the COLMAP format. -Original file at: -https://github.com/colmap/colmap/blob/1a4d0bad2e90aa65ce997c9d1779518eaed998d5/scripts/python/read_write_model.py. -""" +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -# Copyright (c) 2022, ETH Zurich and UNC Chapel Hill. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of -# its contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -# Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) +""" +Tools supporting the execution of COLMAP and preparation of COLMAP-based datasets for nerstudio training. +""" import json -import os -import struct -from dataclasses import dataclass -from io import BufferedReader from pathlib import Path -from typing import Dict, Optional, Tuple +from typing import Dict, Optional import appdirs +import cv2 import numpy as np import requests +import torch from rich.console import Console from rich.progress import track from typing_extensions import Literal +# TODO(1480) use pycolmap instead of colmap_parsing_utils +# import pycolmap +from nerfstudio.data.utils.colmap_parsing_utils import ( + qvec2rotmat, + read_cameras_binary, + read_images_binary, + read_points3d_binary, +) from nerfstudio.process_data.process_data_utils import CameraModel +from nerfstudio.utils import colormaps from nerfstudio.utils.rich_utils import status from nerfstudio.utils.scripts import run_command CONSOLE = Console(width=120) -@dataclass -class ColmapCameraModel: - """Camera model""" - - model_id: int - """Model identifier""" - model_name: str - """Model name""" - num_params: int - """Number of parameters""" - - -@dataclass -class Camera: - """Camera""" - - id: int - """Camera identifier""" - model: str - """Camera model""" - width: int - """Image width""" - height: int - """Image height""" - params: np.ndarray - """Camera parameters""" - - -@dataclass -class Image: - """Data the corresponds to a single image""" - - id: int - """Image identifier""" - qvec: np.ndarray - """Quaternion vector""" - tvec: np.ndarray - """Translation vector""" - camera_id: int - """Camera identifier""" - name: str - """Image name""" - xys: np.ndarray - """2D points""" - point3d_ids: np.ndarray - """Point3D identifiers""" - - -@dataclass -class Point3D: - """Data that corresponds to a single 3D point""" - - id: int - """Point3D identifier""" - xyz: np.ndarray - """3D point""" - rgb: np.ndarray - """Color""" - error: float - """Reconstruction error""" - image_ids: np.ndarray - """Image identifiers""" - point2d_idxs: np.ndarray - """Point2D indices""" - - -COLMAP_CAMERA_MODELS = [ - ColmapCameraModel(model_id=0, model_name="SIMPLE_PINHOLE", num_params=3), - ColmapCameraModel(model_id=1, model_name="PINHOLE", num_params=4), - ColmapCameraModel(model_id=2, model_name="SIMPLE_RADIAL", num_params=4), - ColmapCameraModel(model_id=3, model_name="RADIAL", num_params=5), - ColmapCameraModel(model_id=4, model_name="OPENCV", num_params=8), - ColmapCameraModel(model_id=5, model_name="OPENCV_FISHEYE", num_params=8), - ColmapCameraModel(model_id=6, model_name="FULL_OPENCV", num_params=12), - ColmapCameraModel(model_id=7, model_name="FOV", num_params=5), - ColmapCameraModel(model_id=8, model_name="SIMPLE_RADIAL_FISHEYE", num_params=4), - ColmapCameraModel(model_id=9, model_name="RADIAL_FISHEYE", num_params=5), - ColmapCameraModel(model_id=10, model_name="THIN_PRISM_FISHEYE", num_params=12), -] -COLMAP_CAMERA_MODEL_IDS = {camera_model.model_id: camera_model for camera_model in COLMAP_CAMERA_MODELS} -COLMAP_CAMERA_MODEL_NAMES = {camera_model.model_name: camera_model for camera_model in COLMAP_CAMERA_MODELS} - - def get_colmap_version(colmap_cmd: str, default_version=3.8) -> float: """Returns the version of COLMAP. This code assumes that colmap returns a version string of the form @@ -159,310 +64,6 @@ def get_colmap_version(colmap_cmd: str, default_version=3.8) -> float: return default_version -def read_next_bytes(fid: BufferedReader, num_bytes: int, format_char_sequence, endian_character: str = "<"): - """Read and unpack the next bytes from a binary file. - - Args: - fid: Open file - num_bytes: Sum of combination of {2, 4, 8}, e.g. 2, 6, 16, 30, etc. - format_char_sequence: List of {c, e, f, d, h, H, i, I, l, L, q, Q}. - endian_character: Any of {@, =, <, >, !} - Tuple of read and unpacked values. - """ - data = fid.read(num_bytes) - return struct.unpack(endian_character + format_char_sequence, data) - - -def read_cameras_text(path: Path) -> Dict[int, Camera]: - """Parse COLMAP cameras.txt file into a dictionary of Camera objects. - - Args: - path: Path to cameras.txt file. - Returns: - Dictionary of Camera objects. - """ - cameras = {} - with open(path, encoding="utf-8") as fid: - while True: - line = fid.readline() - if not line: - break - line = line.strip() - if len(line) > 0 and line[0] != "#": - elems = line.split() - camera_id = int(elems[0]) - model = elems[1] - width = int(elems[2]) - height = int(elems[3]) - params = np.array(tuple(map(float, elems[4:]))) - cameras[camera_id] = Camera(id=camera_id, model=model, width=width, height=height, params=params) - return cameras - - -def read_cameras_binary(path_to_model_file: Path) -> Dict[int, Camera]: - """Parse COLMAP cameras.bin file into a dictionary of Camera objects. - - Args: - path_to_model_file: Path to cameras.bin file. - Returns: - Dictionary of Camera objects. - """ - cameras = {} - with open(path_to_model_file, "rb") as fid: - num_cameras = read_next_bytes(fid, 8, "Q")[0] - for _ in range(num_cameras): - camera_properties = read_next_bytes(fid, num_bytes=24, format_char_sequence="iiQQ") - camera_id = camera_properties[0] - model_id = camera_properties[1] - model_name = COLMAP_CAMERA_MODEL_IDS[camera_properties[1]].model_name - width = camera_properties[2] - height = camera_properties[3] - num_params = COLMAP_CAMERA_MODEL_IDS[model_id].num_params - params = read_next_bytes(fid, num_bytes=8 * num_params, format_char_sequence="d" * num_params) - cameras[camera_id] = Camera( - id=camera_id, model=model_name, width=width, height=height, params=np.array(params) - ) - assert len(cameras) == num_cameras - return cameras - - -def read_images_text(path: Path) -> Dict[int, Image]: - """Parse COLMAP images.txt file into a dictionary of Image objects. - - Args: - path: Path to images.txt file. - Returns: - Dictionary of Image objects. - """ - images = {} - with open(path, encoding="utf-8") as fid: - while True: - line = fid.readline() - if not line: - break - line = line.strip() - if len(line) > 0 and line[0] != "#": - elems = line.split() - image_id = int(elems[0]) - qvec = np.array(tuple(map(float, elems[1:5]))) - tvec = np.array(tuple(map(float, elems[5:8]))) - camera_id = int(elems[8]) - image_name = elems[9] - elems = fid.readline().split() - xys = np.column_stack([tuple(map(float, elems[0::3])), tuple(map(float, elems[1::3]))]) - point3d_ids = np.array(tuple(map(int, elems[2::3]))) - images[image_id] = Image( - id=image_id, - qvec=qvec, - tvec=tvec, - camera_id=camera_id, - name=image_name, - xys=xys, - point3d_ids=point3d_ids, - ) - return images - - -def read_images_binary(path_to_model_file: Path) -> Dict[int, Image]: - """Parse COLMAP images.bin file into a dictionary of Image objects. - - Args: - path_to_model_file: Path to images.bin file. - Returns: - Dictionary of Image objects. - """ - images = {} - with open(path_to_model_file, "rb") as fid: - num_reg_images = read_next_bytes(fid, 8, "Q")[0] - for _ in range(num_reg_images): - binary_image_properties = read_next_bytes(fid, num_bytes=64, format_char_sequence="idddddddi") - image_id = binary_image_properties[0] - qvec = np.array(binary_image_properties[1:5]) - tvec = np.array(binary_image_properties[5:8]) - camera_id = binary_image_properties[8] - image_name = "" - current_char = read_next_bytes(fid, 1, "c")[0] - while current_char != b"\x00": # look for the ASCII 0 entry - image_name += current_char.decode("utf-8") - current_char = read_next_bytes(fid, 1, "c")[0] - num_points2d = read_next_bytes(fid, num_bytes=8, format_char_sequence="Q")[0] - x_y_id_s = read_next_bytes(fid, num_bytes=24 * num_points2d, format_char_sequence="ddq" * num_points2d) - xys = np.column_stack([tuple(map(float, x_y_id_s[0::3])), tuple(map(float, x_y_id_s[1::3]))]) - point3d_ids = np.array(tuple(map(int, x_y_id_s[2::3]))) - images[image_id] = Image( - id=image_id, - qvec=qvec, - tvec=tvec, - camera_id=camera_id, - name=image_name, - xys=xys, - point3d_ids=point3d_ids, - ) - return images - - -def read_points3d_text(path) -> Dict[int, Point3D]: - """Parse COLMAP points3D.txt file into a dictionary of Point3D objects. - - Args: - path: Path to points3D.txt file. - Returns: - Dictionary of Point3D objects. - """ - points3d = {} - with open(path, encoding="utf-8") as fid: - while True: - line = fid.readline() - if not line: - break - line = line.strip() - if len(line) > 0 and line[0] != "#": - elems = line.split() - point3d_id = int(elems[0]) - xyz = np.array(tuple(map(float, elems[1:4]))) - rgb = np.array(tuple(map(int, elems[4:7]))) - error = float(elems[7]) - image_ids = np.array(tuple(map(int, elems[8::2]))) - point2d_idxs = np.array(tuple(map(int, elems[9::2]))) - points3d[point3d_id] = Point3D( - id=point3d_id, xyz=xyz, rgb=rgb, error=error, image_ids=image_ids, point2d_idxs=point2d_idxs - ) - return points3d - - -def read_points3d_binary(path_to_model_file: Path) -> Dict[int, Point3D]: - """Parse COLMAP points3D.bin file into a dictionary of Point3D objects. - - Args: - path_to_model_file: Path to points3D.bin file. - Returns: - Dictionary of Point3D objects. - """ - points3d = {} - with open(path_to_model_file, "rb") as fid: - num_points = read_next_bytes(fid, 8, "Q")[0] - for _ in range(num_points): - binary_point_line_properties = read_next_bytes(fid, num_bytes=43, format_char_sequence="QdddBBBd") - point3d_id = binary_point_line_properties[0] - xyz = np.array(binary_point_line_properties[1:4]) - rgb = np.array(binary_point_line_properties[4:7]) - error = np.array(binary_point_line_properties[7]) - track_length = read_next_bytes(fid, num_bytes=8, format_char_sequence="Q")[0] - track_elems = read_next_bytes(fid, num_bytes=8 * track_length, format_char_sequence="ii" * track_length) - image_ids = np.array(tuple(map(int, track_elems[0::2]))) - point2d_idxs = np.array(tuple(map(int, track_elems[1::2]))) - points3d[point3d_id] = Point3D( - id=point3d_id, xyz=xyz, rgb=rgb, error=float(error), image_ids=image_ids, point2d_idxs=point2d_idxs - ) - return points3d - - -def detect_model_format(path: Path, ext: str) -> bool: - """Detect the format of the model file. - - Args: - path: Path to the model file. - ext: Extension to test. - Returns: - True if the model file is the tested extension, False otherwise. - """ - - if ( - os.path.isfile(path / f"cameras{ext}") - and os.path.isfile(path / f"images{ext}") - and os.path.isfile(path / f"points3D{ext}") - ): - print("Detected model format: '" + ext + "'") - return True - - return False - - -def read_model(path: Path, ext: Optional[str] = None) -> Tuple[Dict[int, Camera], Dict[int, Image], Dict[int, Point3D]]: - """Read a COLMAP model from a directory. - - Args: - path: Path to the model directory. - ext: Extension of the model files. If None, the function will try to detect the format. - Returns: - Tuple of dictionaries of Camera, Image, and Point3D objects. - """ - # try to detect the extension automatically - if ext is None: - if detect_model_format(path, ".bin"): - ext = ".bin" - elif detect_model_format(path, ".txt"): - ext = ".txt" - else: - raise ValueError("Provide model format: '.bin' or '.txt'") - - if ext == ".txt": - cameras = read_cameras_text(path / f"cameras{ext}") - images = read_images_text(path / f"images{ext}") - points3d = read_points3d_text(path / f"points3D{ext}") - else: - cameras = read_cameras_binary(path / f"cameras{ext}") - images = read_images_binary(path / f"images{ext}") - points3d = read_points3d_binary(path / f"points3D{ext}") - return cameras, images, points3d - - -def qvec2rotmat(qvec) -> np.ndarray: - """Convert quaternion to rotation matrix. - - Args: - qvec: Quaternion vector of shape (4,). - Returns: - Rotation matrix of shape (3, 3). - """ - return np.array( - [ - [ - 1 - 2 * qvec[2] ** 2 - 2 * qvec[3] ** 2, - 2 * qvec[1] * qvec[2] - 2 * qvec[0] * qvec[3], - 2 * qvec[3] * qvec[1] + 2 * qvec[0] * qvec[2], - ], - [ - 2 * qvec[1] * qvec[2] + 2 * qvec[0] * qvec[3], - 1 - 2 * qvec[1] ** 2 - 2 * qvec[3] ** 2, - 2 * qvec[2] * qvec[3] - 2 * qvec[0] * qvec[1], - ], - [ - 2 * qvec[3] * qvec[1] - 2 * qvec[0] * qvec[2], - 2 * qvec[2] * qvec[3] + 2 * qvec[0] * qvec[1], - 1 - 2 * qvec[1] ** 2 - 2 * qvec[2] ** 2, - ], - ] - ) - - -def rotmat2qvec(R): - """Convert rotation matrix to quaternion. - - Args: - R: Rotation matrix of shape (3, 3). - Returns: - Quaternion vector of shape (4,). - """ - rxx, ryx, rzx, rxy, ryy, rzy, rxz, ryz, rzz = R.flat - K = ( - np.array( - [ - [rxx - ryy - rzz, 0, 0, 0], - [ryx + rxy, ryy - rxx - rzz, 0, 0], - [rzx + rxz, rzy + ryz, rzz - rxx - ryy, 0], - [ryz - rzy, rzx - rxz, rxy - ryx, rxx + ryy + rzz], - ] - ) - / 3.0 - ) - eigvals, eigvecs = np.linalg.eigh(K) - qvec = eigvecs[np.array([3, 0, 1, 2]), np.argmax(eigvals)] - if qvec[0] < 0: - qvec *= -1 - return qvec - - def get_vocab_tree() -> Path: """Return path to vocab tree. Downloads vocab tree if it doesn't exist. @@ -582,40 +183,54 @@ def run_colmap( def colmap_to_json( - cameras_path: Path, - images_path: Path, + recon_dir: Path, output_dir: Path, camera_model: CameraModel, camera_mask_path: Optional[Path] = None, + image_id_to_depth_path: Optional[Dict[int, Path]] = None, image_rename_map: Optional[Dict[str, str]] = None, ) -> int: """Converts COLMAP's cameras.bin and images.bin to a JSON file. Args: - cameras_path: Path to the cameras.bin file. - images_path: Path to the images.bin file. + recon_dir: Path to the reconstruction directory, e.g. "sparse/0" output_dir: Path to the output directory. - camera_mask_path: Path to the camera mask. camera_model: Camera model used. + camera_mask_path: Path to the camera mask. + image_id_to_depth_path: When including sfm-based depth, embed these depth file paths in the exported json + image_rename_map: Use these image names instead of the names embedded in the COLMAP db Returns: The number of registered images. """ - cameras = read_cameras_binary(cameras_path) - images = read_images_binary(images_path) + # TODO(1480) use pycolmap + # recon = pycolmap.Reconstruction(recon_dir) + # cam_id_to_camera = recon.cameras + # im_id_to_image = recon.images + cam_id_to_camera = read_cameras_binary(recon_dir / "cameras.bin") + im_id_to_image = read_images_binary(recon_dir / "images.bin") - # Only supports one camera - camera_params = cameras[1].params + # Only support first camera + CAMERA_ID = 1 # pylint: disable=invalid-name + camera_params = cam_id_to_camera[CAMERA_ID].params frames = [] - for _, im_data in images.items(): + for im_id, im_data in im_id_to_image.items(): + # NB: COLMAP uses Eigen / scalar-first quaternions + # * https://colmap.github.io/format.html + # * https://github.com/colmap/colmap/blob/bf3e19140f491c3042bfd85b7192ef7d249808ec/src/base/pose.cc#L75 + # the `rotation_matrix()` handles that format for us. + + # TODO(1480) BEGIN use pycolmap API + # rotation = im_data.rotation_matrix() rotation = qvec2rotmat(im_data.qvec) + translation = im_data.tvec.reshape(3, 1) w2c = np.concatenate([rotation, translation], 1) w2c = np.concatenate([w2c, np.array([[0, 0, 0, 1]])], 0) c2w = np.linalg.inv(w2c) - # Convert from COLMAP's camera coordinate system to ours + # Convert from COLMAP's camera coordinate system (OpenCV) to ours (OpenGL) c2w[0:3, 1:3] *= -1 c2w = c2w[np.array([1, 0, 2, 3]), :] c2w[2, :] *= -1 @@ -628,9 +243,13 @@ def colmap_to_json( frame = { "file_path": name.as_posix(), "transform_matrix": c2w.tolist(), + "colmap_im_id": im_id, } if camera_mask_path is not None: frame["mask_path"] = camera_mask_path.relative_to(camera_mask_path.parent.parent).as_posix() + if image_id_to_depth_path is not None: + depth_path = image_id_to_depth_path[im_id] + frame["depth_file_path"] = str(depth_path) frames.append(frame) out = { @@ -638,8 +257,8 @@ def colmap_to_json( "fl_y": float(camera_params[1]), "cx": float(camera_params[2]), "cy": float(camera_params[3]), - "w": cameras[1].width, - "h": cameras[1].height, + "w": cam_id_to_camera[CAMERA_ID].width, + "h": cam_id_to_camera[CAMERA_ID].height, "camera_model": camera_model.value, } @@ -670,6 +289,151 @@ def colmap_to_json( return len(frames) +def create_sfm_depth( + recon_dir: Path, + output_dir: Path, + verbose: bool = True, + depth_scale_to_integer_factor: float = 1000.0, + min_depth: float = 0.001, + max_depth: float = 10000, + max_repoj_err: float = 2.5, + min_n_visible: int = 2, + include_depth_debug: bool = False, + input_images_dir: Optional[Path] = None, +) -> Dict[int, Path]: + """Converts COLMAP's points3d.bin to sparse depth map images encoded as + 16-bit "millimeter depth" PNGs. + + Notes: + * This facility does NOT use COLMAP dense reconstruction; it creates depth + maps from sparse SfM points here. + * COLMAP does *not* reconstruct metric depth unless you give it calibrated + (metric) intrinsics as input. Therefore, "depth" in this function has + potentially ambiguous units. + + Args: + recon_dir: Path to the reconstruction directory, e.g. "sparse/0" + output_dir: Path to the output directory. + verbose: If True, logs progress of depth image creation. + depth_scale_to_integer_factor: Use this parameter to tune the conversion of + raw depth measurements to integer depth values. This value should + be equal to 1. / `depth_unit_scale_factor`, where + `depth_unit_scale_factor` is the value you provide at training time. + E.g. for millimeter depth, leave `depth_unit_scale_factor` at 1e-3 + and depth_scale_to_integer_factor at 1000. + min_depth: Discard points closer than this to the camera. + max_depth: Discard points farther than this from the camera. + max_repoj_err: Discard points with reprojection error greater than this + amount (in pixels). + min_n_visible: Discard 3D points that have been triangulated with fewer + than this many frames. + include_depth_debug: Also include debug images showing depth overlaid + upon RGB. + Returns: + Depth file paths indexed by COLMAP image id + """ + + # TODO(1480) use pycolmap + # recon = pycolmap.Reconstruction(recon_dir) + # ptid_to_info = recon.points3D + # cam_id_to_camera = recon.cameras + # im_id_to_image = recon.images + ptid_to_info = read_points3d_binary(recon_dir / "points3D.bin") + cam_id_to_camera = read_cameras_binary(recon_dir / "cameras.bin") + im_id_to_image = read_images_binary(recon_dir / "images.bin") + + # Only support first camera + CAMERA_ID = 1 # pylint: disable=invalid-name + W = cam_id_to_camera[CAMERA_ID].width + H = cam_id_to_camera[CAMERA_ID].height + + if verbose: + iter_images = track( + im_id_to_image.items(), total=len(im_id_to_image.items()), description="Creating depth maps ..." + ) + else: + iter_images = iter(im_id_to_image.items()) + + image_id_to_depth_path = {} + for im_id, im_data in iter_images: + # TODO(1480) BEGIN delete when abandoning colmap_parsing_utils + pids = [pid for pid in im_data.point3D_ids if pid != -1] + xyz_world = np.array([ptid_to_info[pid].xyz for pid in pids]) + rotation = qvec2rotmat(im_data.qvec) + z = (rotation @ xyz_world.T)[-1] + im_data.tvec[-1] + errors = np.array([ptid_to_info[pid].error for pid in pids]) + n_visible = np.array([len(ptid_to_info[pid].image_ids) for pid in pids]) + uv = np.array([im_data.xys[i] for i in range(len(im_data.xys)) if im_data.point3D_ids[i] != -1]) + # TODO(1480) END delete when abandoning colmap_parsing_utils + + # TODO(1480) BEGIN use pycolmap API + + # # Get only keypoints that have corresponding triangulated 3D points + # p2ds = im_data.get_valid_points2D() + + # xyz_world = np.array([ptid_to_info[p2d.point3D_id].xyz for p2d in p2ds]) + + # # COLMAP OpenCV convention: z is always positive + # z = (im_data.rotation_matrix() @ xyz_world.T)[-1] + im_data.tvec[-1] + + # # Mean reprojection error in image space + # errors = np.array([ptid_to_info[p2d.point3D_id].error for p2d in p2ds]) + + # # Number of frames in which each frame is visible + # n_visible = np.array([ptid_to_info[p2d.point3D_id].track.length() for p2d in p2ds]) + + # Note: these are *unrectified* pixel coordinates that should match the original input + # no matter the camera model + # uv = np.array([p2d.xy for p2d in p2ds]) + + # TODO(1480) END use pycolmap API + + idx = np.where( + (z >= min_depth) + & (z <= max_depth) + & (errors <= max_repoj_err) + & (n_visible >= min_n_visible) + & (uv[:, 0] >= 0) + & (uv[:, 0] < W) + & (uv[:, 1] >= 0) + & (uv[:, 1] < H) + ) + z = z[idx] + uv = uv[idx] + + uu, vv = uv[:, 0].astype(int), uv[:, 1].astype(int) + depth = np.zeros((H, W), dtype=np.float32) + depth[vv, uu] = z + + # E.g. if `depth` is metric and in units of meters, and `depth_scale_to_integer_factor` + # is 1000, then `depth_img` will be integer millimeters. + depth_img = (depth_scale_to_integer_factor * depth).astype(np.uint16) + + out_name = str(im_data.name) + depth_path = output_dir / out_name + cv2.imwrite(str(depth_path), depth_img) + + image_id_to_depth_path[im_id] = depth_path + + if include_depth_debug: + assert input_images_dir is not None, "Need explicit input_images_dir for debug images" + assert input_images_dir.exists(), input_images_dir + + depth_flat = depth.flatten()[:, None] + overlay = 255.0 * colormaps.apply_depth_colormap(torch.from_numpy(depth_flat)).numpy() + overlay = overlay.reshape([H, W, 3]) + input_image_path = input_images_dir / im_data.name + input_image = cv2.imread(str(input_image_path)) + debug = 0.3 * input_image + 0.7 + overlay + + out_name = out_name + ".debug.jpg" + output_path = output_dir / "debug_depth" / out_name + output_path.parent.mkdir(parents=True, exist_ok=True) + cv2.imwrite(str(output_path), debug.astype(np.uint8)) + + return image_id_to_depth_path + + def get_matching_summary(num_intial_frames: int, num_matched_frames: int) -> str: """Returns a summary of the matching results. diff --git a/nerfstudio/process_data/hloc_utils.py b/nerfstudio/process_data/hloc_utils.py index 17bd0120c..0f46dd12c 100644 --- a/nerfstudio/process_data/hloc_utils.py +++ b/nerfstudio/process_data/hloc_utils.py @@ -28,6 +28,7 @@ from typing_extensions import Literal from nerfstudio.process_data.process_data_utils import CameraModel try: + # TODO(1480) un-hide pycolmap import import pycolmap from hloc import ( extract_features, @@ -95,14 +96,16 @@ def run_hloc( pairs_from_retrieval.main(retrieval_path, sfm_pairs, num_matched=num_matched) match_features.main(matcher_conf, sfm_pairs, features=features, matches=matches) - image_options = pycolmap.ImageReaderOptions(camera_model=camera_model.value) + image_options = pycolmap.ImageReaderOptions( # pylint: disable=c-extension-no-member + camera_model=camera_model.value + ) reconstruction.main( sfm_dir, image_dir, sfm_pairs, features, matches, - camera_mode=pycolmap.CameraMode.SINGLE, + camera_mode=pycolmap.CameraMode.SINGLE, # pylint: disable=c-extension-no-member image_options=image_options, verbose=verbose, ) diff --git a/nerfstudio/process_data/polycam_utils.py b/nerfstudio/process_data/polycam_utils.py index 079ff16f7..a101c5062 100644 --- a/nerfstudio/process_data/polycam_utils.py +++ b/nerfstudio/process_data/polycam_utils.py @@ -216,7 +216,9 @@ def process_depth_maps( # Downscale depth maps summary_log.append( - process_data_utils.downscale_images(depth_dir, num_downscales, folder_name="depth", verbose=verbose) + process_data_utils.downscale_images( + depth_dir, num_downscales, folder_name="depths", nearest_neighbor=True, verbose=verbose + ) ) return summary_log, polycam_depth_maps_filenames diff --git a/nerfstudio/process_data/process_data_utils.py b/nerfstudio/process_data/process_data_utils.py index b1929deaa..31456d94a 100644 --- a/nerfstudio/process_data/process_data_utils.py +++ b/nerfstudio/process_data/process_data_utils.py @@ -263,7 +263,13 @@ def copy_images(data: Path, image_dir: Path, verbose) -> OrderedDict[Path, Path] return OrderedDict((original_path, new_path) for original_path, new_path in zip(image_paths, copied_images)) -def downscale_images(image_dir: Path, num_downscales: int, folder_name: str = "images", verbose: bool = False) -> str: +def downscale_images( + image_dir: Path, + num_downscales: int, + folder_name: str = "images", + nearest_neighbor: bool = False, + verbose: bool = False, +) -> str: """Downscales the images in the directory. Uses FFMPEG. Assumes images are named frame_00001.png, frame_00002.png, etc. @@ -272,6 +278,7 @@ def downscale_images(image_dir: Path, num_downscales: int, folder_name: str = "i image_dir: Path to the directory containing the images. num_downscales: Number of times to downscale the images. Downscales by 2 each time. folder_name: Name of the output folder + nearest_neighbor: Use nearest neighbor sampling (useful for depth images) verbose: If True, logs the output of the command. Returns: @@ -291,10 +298,13 @@ def downscale_images(image_dir: Path, num_downscales: int, folder_name: str = "i # Using %05d ffmpeg commands appears to be unreliable (skips images), so use scandir. files = os.scandir(image_dir) for f in files: + if f.is_dir(): + continue filename = f.name + nn_flag = "" if not nearest_neighbor else ":flags=neighbor" ffmpeg_cmd = [ f'ffmpeg -y -noautorotate -i "{image_dir / filename}" ', - f"-q:v 2 -vf scale=iw/{downscale_factor}:ih/{downscale_factor} ", + f"-q:v 2 -vf scale=iw/{downscale_factor}:ih/{downscale_factor}{nn_flag} ", f'"{downscale_dir / filename}"', ] ffmpeg_cmd = " ".join(ffmpeg_cmd) diff --git a/nerfstudio/process_data/record3d_utils.py b/nerfstudio/process_data/record3d_utils.py index d24292817..ed7393086 100644 --- a/nerfstudio/process_data/record3d_utils.py +++ b/nerfstudio/process_data/record3d_utils.py @@ -46,6 +46,8 @@ def record3d_to_json(images_paths: List[Path], metadata_path: Path, output_dir: metadata_dict = io.load_from_json(metadata_path) poses_data = np.array(metadata_dict["poses"]) # (N, 3, 4) + # NB: Record3D / scipy use "scalar-last" format quaternions (x y z w) + # https://fzheng.me/2017/11/12/quaternion_conventions_en/ camera_to_worlds = np.concatenate( [Rotation.from_quat(poses_data[:, :4]).as_matrix(), poses_data[:, 4:, None]], axis=-1, diff --git a/nerfstudio/utils/colormaps.py b/nerfstudio/utils/colormaps.py index 18d7aa86f..aae1b2a92 100644 --- a/nerfstudio/utils/colormaps.py +++ b/nerfstudio/utils/colormaps.py @@ -31,7 +31,7 @@ def apply_colormap(image: TensorType["bs":..., 1], cmap="viridis") -> TensorType cmap: Colormap for image. Returns: - TensorType: Colored image + TensorType: Colored image with colors in [0, 1] """ colormap = cm.get_cmap(cmap) @@ -62,7 +62,7 @@ def apply_depth_colormap( cmap: Colormap to apply. Returns: - Colored depth image + Colored depth image with colors in [0, 1] """ near_plane = near_plane or float(torch.min(depth)) diff --git a/nerfstudio/utils/misc.py b/nerfstudio/utils/misc.py index 5c4541426..88a76e8fc 100644 --- a/nerfstudio/utils/misc.py +++ b/nerfstudio/utils/misc.py @@ -16,6 +16,7 @@ Miscellaneous helper code. """ + from typing import Any, Callable, Dict, List, Optional, Union import torch @@ -147,3 +148,10 @@ def update_avg(prev_avg: float, new_val: float, step: int) -> float: float: new updated average """ return (step * prev_avg + new_val) / (step + 1) + + +def strtobool(val) -> bool: + """Cheap replacement for `distutils.util.strtobool()` which is deprecated + FMI https://stackoverflow.com/a/715468 + """ + return val.lower() in ("yes", "y", "true", "t", "on", "1") diff --git a/pyproject.toml b/pyproject.toml index 8725c0222..314b22141 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,10 +38,13 @@ dependencies = [ "Pillow>=9.3.0", "plotly>=5.7.0", "protobuf<=3.20.3,!=3.20.0", + # TODO(1480) enable when pycolmap windows wheels are available + # "pycolmap==0.3.0", "pyequilib>=0.5.6", "pymeshlab>=2022.2.post2", "pyngrok>=5.1.0", "python-socketio>=5.7.1", + "pyquaternion>=0.9.9", "requests", "rich>=12.5.1", "scikit-image>=0.19.3", diff --git a/scripts/process_data.py b/scripts/process_data.py index b95146d99..53ac9c0da 100644 --- a/scripts/process_data.py +++ b/scripts/process_data.py @@ -91,10 +91,14 @@ class ProcessImages: """ gpu: bool = True """If True, use GPU.""" + use_sfm_depth: bool = False + """If True, export and use depth maps induced from SfM points.""" + include_depth_debug: bool = False + """If --use-sfm-depth and this flag is True, also export debug images showing SfM overlaid upon input images.""" verbose: bool = False """If True, print extra logging.""" - def main(self) -> None: + def main(self) -> None: # pylint: disable=R0915 """Process images into a nerfstudio dataset.""" # pylint: disable=too-many-statements require_cameras_exist = False @@ -161,14 +165,33 @@ class ProcessImages: # Colmap uses renamed images image_rename_map = None + # Export depth maps + if self.use_sfm_depth: + depth_dir = self.output_dir / "depth" + depth_dir.mkdir(parents=True, exist_ok=True) + image_id_to_depth_path = colmap_utils.create_sfm_depth( + recon_dir=colmap_dir / "sparse" / "0", + output_dir=depth_dir, + include_depth_debug=self.include_depth_debug, + input_images_dir=image_dir, + verbose=self.verbose, + ) + summary_log.append( + process_data_utils.downscale_images( + depth_dir, self.num_downscales, folder_name="depths", nearest_neighbor=True, verbose=self.verbose + ) + ) + else: + image_id_to_depth_path = {} + # Save transforms.json if (colmap_model_path / "cameras.bin").exists(): with CONSOLE.status("[bold yellow]Saving results to transforms.json", spinner="balloon"): num_matched_frames = colmap_utils.colmap_to_json( - cameras_path=colmap_model_path / "cameras.bin", - images_path=colmap_model_path / "images.bin", + recon_dir=colmap_dir / "sparse" / "0", output_dir=self.output_dir, camera_model=CAMERA_MODELS[self.camera_type], + image_id_to_depth_path=image_id_to_depth_path, image_rename_map=image_rename_map, ) summary_log.append(f"Colmap matched {num_matched_frames} images") @@ -272,12 +295,16 @@ class ProcessVideo: """Create circle crop mask. The radius is the percent of the image diagonal.""" percent_crop: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0) """Percent of the image to crop. (top, bottom, left, right)""" + use_sfm_depth: bool = False + """If True, export and use depth maps induced from SfM points.""" + include_depth_debug: bool = False + """If --use-sfm-depth and this flag is True, also export debug images showing SfM overlaid upon input images.""" gpu: bool = True """If True, use GPU.""" verbose: bool = False """If True, print extra logging.""" - def main(self) -> None: + def main(self) -> None: # pylint: disable=R0915 """Process video into a nerfstudio dataset.""" install_checks.check_ffmpeg_installed() install_checks.check_colmap_installed() @@ -372,14 +399,33 @@ class ProcessVideo: CONSOLE.log("[bold red]Invalid combination of sfm_tool, feature_type, and matcher_type, exiting") sys.exit(1) + # Export depth maps + if self.use_sfm_depth: + depth_dir = self.output_dir / "depth" + depth_dir.mkdir(parents=True, exist_ok=True) + image_id_to_depth_path = colmap_utils.create_sfm_depth( + recon_dir=colmap_dir / "sparse" / "0", + output_dir=depth_dir, + include_depth_debug=self.include_depth_debug, + input_images_dir=image_dir, + verbose=self.verbose, + ) + summary_log.append( + process_data_utils.downscale_images( + depth_dir, self.num_downscales, folder_name="depths", nearest_neighbor=True, verbose=self.verbose + ) + ) + else: + image_id_to_depth_path = {} + # Save transforms.json if (colmap_dir / "sparse" / "0" / "cameras.bin").exists(): with CONSOLE.status("[bold yellow]Saving results to transforms.json", spinner="balloon"): num_matched_frames = colmap_utils.colmap_to_json( - cameras_path=colmap_dir / "sparse" / "0" / "cameras.bin", - images_path=colmap_dir / "sparse" / "0" / "images.bin", + recon_dir=colmap_dir / "sparse" / "0", output_dir=self.output_dir, camera_model=CAMERA_MODELS[self.camera_type], + image_id_to_depth_path=image_id_to_depth_path, camera_mask_path=mask_path, image_rename_map=None, ) @@ -426,6 +472,10 @@ class ProcessInsta360: """If True, skips COLMAP and generates transforms.json if possible.""" colmap_cmd: str = "colmap" """How to call the COLMAP executable.""" + use_sfm_depth: bool = False + """If True, export and use depth maps induced from SfM points.""" + include_depth_debug: bool = False + """If --use-sfm-depth and this flag is True, also export debug images showing SfM overlaid upon input images.""" gpu: bool = True """If True, use GPU.""" verbose: bool = False @@ -493,14 +543,33 @@ class ProcessInsta360: colmap_cmd=self.colmap_cmd, ) + # Export depth maps + if self.use_sfm_depth: + depth_dir = self.output_dir / "depth" + depth_dir.mkdir(parents=True, exist_ok=True) + image_id_to_depth_path = colmap_utils.create_sfm_depth( + recon_dir=colmap_dir / "sparse" / "0", + output_dir=depth_dir, + include_depth_debug=self.include_depth_debug, + input_images_dir=image_dir, + verbose=self.verbose, + ) + summary_log.append( + process_data_utils.downscale_images( + depth_dir, self.num_downscales, folder_name="depths", nearest_neighbor=True, verbose=self.verbose + ) + ) + else: + image_id_to_depth_path = {} + # Save transforms.json if (colmap_dir / "sparse" / "0" / "cameras.bin").exists(): with CONSOLE.status("[bold yellow]Saving results to transforms.json", spinner="balloon"): num_matched_frames = colmap_utils.colmap_to_json( - cameras_path=colmap_dir / "sparse" / "0" / "cameras.bin", - images_path=colmap_dir / "sparse" / "0" / "images.bin", + recon_dir=colmap_dir / "sparse" / "0", output_dir=self.output_dir, camera_model=CAMERA_MODELS["fisheye"], + image_id_to_depth_path=image_id_to_depth_path, image_rename_map=None, ) summary_log.append(f"Colmap matched {num_matched_frames} images") diff --git a/tests/process_data/test_misc.py b/tests/process_data/test_misc.py new file mode 100644 index 000000000..a300687ee --- /dev/null +++ b/tests/process_data/test_misc.py @@ -0,0 +1,63 @@ +""" +Test misc data utils +""" + +import numpy as np +from pyquaternion import Quaternion +from scipy.spatial.transform import Rotation + +# TODO(1480) use pycolmap instead of colmap_parsing_utils +# import pycolmap +from nerfstudio.data.utils.colmap_parsing_utils import qvec2rotmat + + +def test_scalar_first_scalar_last_quaternions(): + """Many nerfstudio datasets use quaternions for pose rotations. Some use + scalar-first format and others use scalar-last. However internally + nerfstudio uses rotation *matrices* / projection matrices. This test + helps document differences in quaternion encodings & conversion to + rotation matrices. + + FMI see call sites in nerfstudio where the various libraries are used. You + might also consult this detailed reference: + * "Why and How to Avoid the Flipped Quaternion Multiplication" + Sommer et al. + https://arxiv.org/abs/1801.07478 + """ + + # Pick an arbitrary euler rotation; note that different communities + # also use different rotation axes (e.g. Euler vs Tait-Bryan ...) + rot = Rotation.from_euler("xyz", np.array([10.0, 20.0, 30.0]), degrees=True) + + # scalar-last + xyzw = rot.as_quat() + assert np.allclose(xyzw, np.array([0.03813458, 0.18930786, 0.23929834, 0.95154852])) + + # scalar-first + wxyz = xyzw[[3, 0, 1, 2]] + assert np.allclose(wxyz, np.array([0.95154852, 0.03813458, 0.18930786, 0.23929834])) + + # Expected Rotation matrix + # fmt: off + R_expected = np.array( # pylint: disable=invalid-name + [ + [ 0.81379768, -0.44096961, 0.37852231], + [ 0.46984631, 0.88256412, 0.01802831], + [-0.34202014, 0.16317591, 0.92541658] + ] + ) + # fmt: on + + # Record3D / scipy + R = Rotation.from_quat(xyzw).as_matrix() + assert np.allclose(R, R_expected) + + # Nuscenes / pyquaternion + R = Quaternion(wxyz).rotation_matrix + assert np.allclose(R, R_expected) + + # COLMAP + # TODO(1480) use pycolmap + # R = pycolmap.qvec_to_rotmat(wxyz) + R = qvec2rotmat(wxyz) + assert np.allclose(R, R_expected) diff --git a/tests/utils/test_aabb_intersection.py b/tests/utils/test_aabb_intersection.py index 1e01dae0b..1d6ba34b9 100644 --- a/tests/utils/test_aabb_intersection.py +++ b/tests/utils/test_aabb_intersection.py @@ -5,11 +5,11 @@ import importlib import math import os import random -from distutils.util import strtobool import torch from nerfstudio.utils.math import intersect_aabb +from nerfstudio.utils.misc import strtobool test_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")