SFM: sfm-based depth, some breadcrumbs for pycolmap (#1457)

* save

* save

* more cleanup

* save

* save

* update phototourism

* save

* works

* works

* cleanup

* cleanup

* lint

* python3 default

* cleanup
This commit is contained in:
Paul Wais authored and GitHub committed 2023-02-25 21:10:53 -08:00
1 parent b62571d537
commit 5c73766653
19 files changed
+621 -469

No files matched your search

+190
View File
@@ -0,0 +1,190 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
!scripts/downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/#use-with-ide
.pdm.toml
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.envrc
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
# Experiments and outputs
outputs/
exports/
renders/
# tensorboard log files
events.out.*
# Data
data
# Misc
old/
temp*
!temporal*
.nfs*
external/
__MACOSX/
node_modules/
bash/
cache/
package-lock.json
camera_paths/
.DS_Store
._.DS_Store
*/**/.DS_Store
*/**/._.DS_Store
+5
View File
@@ -184,3 +184,8 @@ bash/
cache/
package-lock.json
camera_paths/
.DS_Store
._.DS_Store
*/**/.DS_Store
*/**/._.DS_Store
+9 -7
View File
@@ -48,10 +48,12 @@ RUN apt-get update && \
libsuitesparse-dev \
nano \
protobuf-compiler \
python-is-python3 \
python3.10-dev \
python3-pip \
qtbase5-dev \
sudo \
vim-tiny \
wget && \
rm -rf /var/lib/apt/lists/*
@@ -62,7 +64,7 @@ RUN git clone --branch v0.6.0 https://github.com/google/glog.git --single-branch
mkdir build && \
cd build && \
cmake .. && \
make -j && \
make -j `nproc` && \
make install && \
cd ../.. && \
rm -rf glog
@@ -76,7 +78,7 @@ RUN git clone --branch 2.1.0 https://ceres-solver.googlesource.com/ceres-solver.
mkdir build && \
cd build && \
cmake .. -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF && \
make -j && \
make -j `nproc` && \
make install && \
cd ../.. && \
rm -rf ceres-solver
@@ -87,13 +89,12 @@ RUN git clone --branch 3.8 https://github.com/colmap/colmap.git --single-branch
mkdir build && \
cd build && \
cmake .. -DCUDA_ENABLED=ON \
-DCUDA_NVCC_FLAGS="--std c++14" \
-DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES} && \
make -j && \
make -j `nproc` && \
make install && \
cd ../.. && \
rm -rf colmap
# Create non root user and setup environment.
RUN useradd -m -d /home/user -g root -G sudo -u 1000 user
RUN usermod -aG sudo user
@@ -112,13 +113,14 @@ SHELL ["/bin/bash", "-c"]
# Upgrade pip and install packages.
RUN python3.10 -m pip install --upgrade pip setuptools pathtools promise pybind11
# Install pytorch and submodules (Currently, we still use cu116 which is the latest version for toch 1.12.1 and is compatible with CUDA 11.8).
# Install pytorch and submodules (Currently, we still use cu116 which is the latest version for torch 1.12.1 and is compatible with CUDA 11.8).
RUN python3.10 -m pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116
# Install tynyCUDNN (we need to set the target architectures as environment variable first).
ENV TCNN_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES}
RUN python3.10 -m pip install git+https://github.com/NVlabs/tiny-cuda-nn.git#subdirectory=bindings/torch
RUN python3.10 -m pip install git+https://github.com/NVlabs/tiny-cuda-nn.git@v1.6#subdirectory=bindings/torch
# Install pycolmap 0.3.0, required by hloc.
# TODO(https://github.com/colmap/pycolmap/issues/111) use wheel when available for Python 3.10
RUN git clone --branch v0.3.0 --recursive https://github.com/colmap/pycolmap.git && \
cd pycolmap && \
python3.10 -m pip install . && \
+8
View File
@@ -135,6 +135,14 @@ or be built from the repository using
```bash
docker build --tag nerfstudio -f Dockerfile .
```
To restrict to only CUDA architectures that you have available locally, use the `CUDA_ARCHITECTURES`
build arg and look up [the compute capability for your GPU](https://developer.nvidia.com/cuda-gpus).
For example, here's how to build with support for GeForce 30xx series GPUs:
```bash
docker build --build-arg CUDA_ARCHITECTURES=86 -tag nerfstudio-86 .
```
### Using an interactive container
The docker container can be launched with an interactive terminal where nerfstudio commands can be entered as usual. Some parameters are required and some are strongly recommended for usage as following:
```bash
+1 -1
View File
@@ -19,7 +19,6 @@ import base64
import math
import os
from dataclasses import dataclass
from distutils.util import strtobool
from enum import Enum, auto
from typing import Dict, List, Optional, Tuple, Union
@@ -34,6 +33,7 @@ import nerfstudio.utils.poses as pose_utils
from nerfstudio.cameras import camera_utils
from nerfstudio.cameras.rays import RayBundle
from nerfstudio.data.scene_box import SceneBox
from nerfstudio.utils.misc import strtobool
from nerfstudio.utils.tensor_dataclass import TensorDataclass
@@ -20,6 +20,7 @@ from pathlib import Path
from typing import Optional, Tuple, Type
import numpy as np
import pyquaternion
import torch
from nuscenes.nuscenes import NuScenes as NuScenesDatabase
from typing_extensions import Literal
@@ -31,14 +32,19 @@ from nerfstudio.data.dataparsers.base_dataparser import (
DataparserOutputs,
)
from nerfstudio.data.scene_box import SceneBox
from nerfstudio.process_data.colmap_utils import qvec2rotmat
def rotation_translation_to_pose(r_vec, t_vec):
def rotation_translation_to_pose(r_quat, t_vec):
"""Convert quaternion rotation and translation vectors to 4x4 matrix"""
pose = np.eye(4)
pose[:3, :3] = qvec2rotmat(r_vec)
# NB: Nuscenes recommends pyquaternion, which uses scalar-first format (w x y z)
# https://github.com/nutonomy/nuscenes-devkit/issues/545#issuecomment-766509242
# https://github.com/KieranWynn/pyquaternion/blob/99025c17bab1c55265d61add13375433b35251af/pyquaternion/quaternion.py#L299
# https://fzheng.me/2017/11/12/quaternion_conventions_en/
pose[:3, :3] = pyquaternion.Quaternion(r_quat).rotation_matrix
pose[:3, 3] = t_vec
return pose
@@ -33,7 +33,13 @@ from nerfstudio.data.dataparsers.base_dataparser import (
DataparserOutputs,
)
from nerfstudio.data.scene_box import SceneBox
from nerfstudio.data.utils.colmap_utils import read_cameras_binary, read_images_binary
# TODO(1480) use pycolmap instead of colmap_parsing_utils
# import pycolmap
from nerfstudio.data.utils.colmap_parsing_utils import (
read_cameras_binary,
read_images_binary,
)
CONSOLE = Console(width=120)
@@ -76,11 +82,14 @@ class Phototourism(DataParser):
# pylint: disable=too-many-statements
def _generate_dataparser_outputs(self, split="train"):
image_filenames = []
poses = []
with CONSOLE.status(f"[bold green]Reading phototourism images and poses for {split} split...") as _:
# TODO(1480) use pycolmap
# recon = pycolmap.Reconstruction(self.data / "dense" / "sparse")
# cams = recon.cameras
# imgs = recon.images
cams = read_cameras_binary(self.data / "dense/sparse/cameras.bin")
imgs = read_images_binary(self.data / "dense/sparse/images.bin")
@@ -1,3 +1,11 @@
"""
This file copied with small modifications from:
* https://github.com/colmap/colmap/blob/1a4d0bad2e90aa65ce997c9d1779518eaed998d5/scripts/python/read_write_model.py
TODO(1480) Delete this file when moving to pycolmap.
"""
# Copyright (c) 2018, ETH Zurich and UNC Chapel Hill.
# All rights reserved.
#
+204 -440
View File
@@ -1,145 +1,50 @@
"""
Here we have modified code taken from COLMAP for parsing data in the COLMAP format.
Original file at:
https://github.com/colmap/colmap/blob/1a4d0bad2e90aa65ce997c9d1779518eaed998d5/scripts/python/read_write_model.py.
"""
# Copyright 2022 The Nerfstudio Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Copyright (c) 2022, ETH Zurich and UNC Chapel Hill.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of
# its contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de)
"""
Tools supporting the execution of COLMAP and preparation of COLMAP-based datasets for nerstudio training.
"""
import json
import os
import struct
from dataclasses import dataclass
from io import BufferedReader
from pathlib import Path
from typing import Dict, Optional, Tuple
from typing import Dict, Optional
import appdirs
import cv2
import numpy as np
import requests
import torch
from rich.console import Console
from rich.progress import track
from typing_extensions import Literal
# TODO(1480) use pycolmap instead of colmap_parsing_utils
# import pycolmap
from nerfstudio.data.utils.colmap_parsing_utils import (
qvec2rotmat,
read_cameras_binary,
read_images_binary,
read_points3d_binary,
)
from nerfstudio.process_data.process_data_utils import CameraModel
from nerfstudio.utils import colormaps
from nerfstudio.utils.rich_utils import status
from nerfstudio.utils.scripts import run_command
CONSOLE = Console(width=120)
@dataclass
class ColmapCameraModel:
"""Camera model"""
model_id: int
"""Model identifier"""
model_name: str
"""Model name"""
num_params: int
"""Number of parameters"""
@dataclass
class Camera:
"""Camera"""
id: int
"""Camera identifier"""
model: str
"""Camera model"""
width: int
"""Image width"""
height: int
"""Image height"""
params: np.ndarray
"""Camera parameters"""
@dataclass
class Image:
"""Data the corresponds to a single image"""
id: int
"""Image identifier"""
qvec: np.ndarray
"""Quaternion vector"""
tvec: np.ndarray
"""Translation vector"""
camera_id: int
"""Camera identifier"""
name: str
"""Image name"""
xys: np.ndarray
"""2D points"""
point3d_ids: np.ndarray
"""Point3D identifiers"""
@dataclass
class Point3D:
"""Data that corresponds to a single 3D point"""
id: int
"""Point3D identifier"""
xyz: np.ndarray
"""3D point"""
rgb: np.ndarray
"""Color"""
error: float
"""Reconstruction error"""
image_ids: np.ndarray
"""Image identifiers"""
point2d_idxs: np.ndarray
"""Point2D indices"""
COLMAP_CAMERA_MODELS = [
ColmapCameraModel(model_id=0, model_name="SIMPLE_PINHOLE", num_params=3),
ColmapCameraModel(model_id=1, model_name="PINHOLE", num_params=4),
ColmapCameraModel(model_id=2, model_name="SIMPLE_RADIAL", num_params=4),
ColmapCameraModel(model_id=3, model_name="RADIAL", num_params=5),
ColmapCameraModel(model_id=4, model_name="OPENCV", num_params=8),
ColmapCameraModel(model_id=5, model_name="OPENCV_FISHEYE", num_params=8),
ColmapCameraModel(model_id=6, model_name="FULL_OPENCV", num_params=12),
ColmapCameraModel(model_id=7, model_name="FOV", num_params=5),
ColmapCameraModel(model_id=8, model_name="SIMPLE_RADIAL_FISHEYE", num_params=4),
ColmapCameraModel(model_id=9, model_name="RADIAL_FISHEYE", num_params=5),
ColmapCameraModel(model_id=10, model_name="THIN_PRISM_FISHEYE", num_params=12),
]
COLMAP_CAMERA_MODEL_IDS = {camera_model.model_id: camera_model for camera_model in COLMAP_CAMERA_MODELS}
COLMAP_CAMERA_MODEL_NAMES = {camera_model.model_name: camera_model for camera_model in COLMAP_CAMERA_MODELS}
def get_colmap_version(colmap_cmd: str, default_version=3.8) -> float:
"""Returns the version of COLMAP.
This code assumes that colmap returns a version string of the form
@@ -159,310 +64,6 @@ def get_colmap_version(colmap_cmd: str, default_version=3.8) -> float:
return default_version
def read_next_bytes(fid: BufferedReader, num_bytes: int, format_char_sequence, endian_character: str = "<"):
"""Read and unpack the next bytes from a binary file.
Args:
fid: Open file
num_bytes: Sum of combination of {2, 4, 8}, e.g. 2, 6, 16, 30, etc.
format_char_sequence: List of {c, e, f, d, h, H, i, I, l, L, q, Q}.
endian_character: Any of {@, =, <, >, !}
Tuple of read and unpacked values.
"""
data = fid.read(num_bytes)
return struct.unpack(endian_character + format_char_sequence, data)
def read_cameras_text(path: Path) -> Dict[int, Camera]:
"""Parse COLMAP cameras.txt file into a dictionary of Camera objects.
Args:
path: Path to cameras.txt file.
Returns:
Dictionary of Camera objects.
"""
cameras = {}
with open(path, encoding="utf-8") as fid:
while True:
line = fid.readline()
if not line:
break
line = line.strip()
if len(line) > 0 and line[0] != "#":
elems = line.split()
camera_id = int(elems[0])
model = elems[1]
width = int(elems[2])
height = int(elems[3])
params = np.array(tuple(map(float, elems[4:])))
cameras[camera_id] = Camera(id=camera_id, model=model, width=width, height=height, params=params)
return cameras
def read_cameras_binary(path_to_model_file: Path) -> Dict[int, Camera]:
"""Parse COLMAP cameras.bin file into a dictionary of Camera objects.
Args:
path_to_model_file: Path to cameras.bin file.
Returns:
Dictionary of Camera objects.
"""
cameras = {}
with open(path_to_model_file, "rb") as fid:
num_cameras = read_next_bytes(fid, 8, "Q")[0]
for _ in range(num_cameras):
camera_properties = read_next_bytes(fid, num_bytes=24, format_char_sequence="iiQQ")
camera_id = camera_properties[0]
model_id = camera_properties[1]
model_name = COLMAP_CAMERA_MODEL_IDS[camera_properties[1]].model_name
width = camera_properties[2]
height = camera_properties[3]
num_params = COLMAP_CAMERA_MODEL_IDS[model_id].num_params
params = read_next_bytes(fid, num_bytes=8 * num_params, format_char_sequence="d" * num_params)
cameras[camera_id] = Camera(
id=camera_id, model=model_name, width=width, height=height, params=np.array(params)
)
assert len(cameras) == num_cameras
return cameras
def read_images_text(path: Path) -> Dict[int, Image]:
"""Parse COLMAP images.txt file into a dictionary of Image objects.
Args:
path: Path to images.txt file.
Returns:
Dictionary of Image objects.
"""
images = {}
with open(path, encoding="utf-8") as fid:
while True:
line = fid.readline()
if not line:
break
line = line.strip()
if len(line) > 0 and line[0] != "#":
elems = line.split()
image_id = int(elems[0])
qvec = np.array(tuple(map(float, elems[1:5])))
tvec = np.array(tuple(map(float, elems[5:8])))
camera_id = int(elems[8])
image_name = elems[9]
elems = fid.readline().split()
xys = np.column_stack([tuple(map(float, elems[0::3])), tuple(map(float, elems[1::3]))])
point3d_ids = np.array(tuple(map(int, elems[2::3])))
images[image_id] = Image(
id=image_id,
qvec=qvec,
tvec=tvec,
camera_id=camera_id,
name=image_name,
xys=xys,
point3d_ids=point3d_ids,
)
return images
def read_images_binary(path_to_model_file: Path) -> Dict[int, Image]:
"""Parse COLMAP images.bin file into a dictionary of Image objects.
Args:
path_to_model_file: Path to images.bin file.
Returns:
Dictionary of Image objects.
"""
images = {}
with open(path_to_model_file, "rb") as fid:
num_reg_images = read_next_bytes(fid, 8, "Q")[0]
for _ in range(num_reg_images):
binary_image_properties = read_next_bytes(fid, num_bytes=64, format_char_sequence="idddddddi")
image_id = binary_image_properties[0]
qvec = np.array(binary_image_properties[1:5])
tvec = np.array(binary_image_properties[5:8])
camera_id = binary_image_properties[8]
image_name = ""
current_char = read_next_bytes(fid, 1, "c")[0]
while current_char != b"\x00": # look for the ASCII 0 entry
image_name += current_char.decode("utf-8")
current_char = read_next_bytes(fid, 1, "c")[0]
num_points2d = read_next_bytes(fid, num_bytes=8, format_char_sequence="Q")[0]
x_y_id_s = read_next_bytes(fid, num_bytes=24 * num_points2d, format_char_sequence="ddq" * num_points2d)
xys = np.column_stack([tuple(map(float, x_y_id_s[0::3])), tuple(map(float, x_y_id_s[1::3]))])
point3d_ids = np.array(tuple(map(int, x_y_id_s[2::3])))
images[image_id] = Image(
id=image_id,
qvec=qvec,
tvec=tvec,
camera_id=camera_id,
name=image_name,
xys=xys,
point3d_ids=point3d_ids,
)
return images
def read_points3d_text(path) -> Dict[int, Point3D]:
"""Parse COLMAP points3D.txt file into a dictionary of Point3D objects.
Args:
path: Path to points3D.txt file.
Returns:
Dictionary of Point3D objects.
"""
points3d = {}
with open(path, encoding="utf-8") as fid:
while True:
line = fid.readline()
if not line:
break
line = line.strip()
if len(line) > 0 and line[0] != "#":
elems = line.split()
point3d_id = int(elems[0])
xyz = np.array(tuple(map(float, elems[1:4])))
rgb = np.array(tuple(map(int, elems[4:7])))
error = float(elems[7])
image_ids = np.array(tuple(map(int, elems[8::2])))
point2d_idxs = np.array(tuple(map(int, elems[9::2])))
points3d[point3d_id] = Point3D(
id=point3d_id, xyz=xyz, rgb=rgb, error=error, image_ids=image_ids, point2d_idxs=point2d_idxs
)
return points3d
def read_points3d_binary(path_to_model_file: Path) -> Dict[int, Point3D]:
"""Parse COLMAP points3D.bin file into a dictionary of Point3D objects.
Args:
path_to_model_file: Path to points3D.bin file.
Returns:
Dictionary of Point3D objects.
"""
points3d = {}
with open(path_to_model_file, "rb") as fid:
num_points = read_next_bytes(fid, 8, "Q")[0]
for _ in range(num_points):
binary_point_line_properties = read_next_bytes(fid, num_bytes=43, format_char_sequence="QdddBBBd")
point3d_id = binary_point_line_properties[0]
xyz = np.array(binary_point_line_properties[1:4])
rgb = np.array(binary_point_line_properties[4:7])
error = np.array(binary_point_line_properties[7])
track_length = read_next_bytes(fid, num_bytes=8, format_char_sequence="Q")[0]
track_elems = read_next_bytes(fid, num_bytes=8 * track_length, format_char_sequence="ii" * track_length)
image_ids = np.array(tuple(map(int, track_elems[0::2])))
point2d_idxs = np.array(tuple(map(int, track_elems[1::2])))
points3d[point3d_id] = Point3D(
id=point3d_id, xyz=xyz, rgb=rgb, error=float(error), image_ids=image_ids, point2d_idxs=point2d_idxs
)
return points3d
def detect_model_format(path: Path, ext: str) -> bool:
"""Detect the format of the model file.
Args:
path: Path to the model file.
ext: Extension to test.
Returns:
True if the model file is the tested extension, False otherwise.
"""
if (
os.path.isfile(path / f"cameras{ext}")
and os.path.isfile(path / f"images{ext}")
and os.path.isfile(path / f"points3D{ext}")
):
print("Detected model format: '" + ext + "'")
return True
return False
def read_model(path: Path, ext: Optional[str] = None) -> Tuple[Dict[int, Camera], Dict[int, Image], Dict[int, Point3D]]:
"""Read a COLMAP model from a directory.
Args:
path: Path to the model directory.
ext: Extension of the model files. If None, the function will try to detect the format.
Returns:
Tuple of dictionaries of Camera, Image, and Point3D objects.
"""
# try to detect the extension automatically
if ext is None:
if detect_model_format(path, ".bin"):
ext = ".bin"
elif detect_model_format(path, ".txt"):
ext = ".txt"
else:
raise ValueError("Provide model format: '.bin' or '.txt'")
if ext == ".txt":
cameras = read_cameras_text(path / f"cameras{ext}")
images = read_images_text(path / f"images{ext}")
points3d = read_points3d_text(path / f"points3D{ext}")
else:
cameras = read_cameras_binary(path / f"cameras{ext}")
images = read_images_binary(path / f"images{ext}")
points3d = read_points3d_binary(path / f"points3D{ext}")
return cameras, images, points3d
def qvec2rotmat(qvec) -> np.ndarray:
"""Convert quaternion to rotation matrix.
Args:
qvec: Quaternion vector of shape (4,).
Returns:
Rotation matrix of shape (3, 3).
"""
return np.array(
[
[
1 - 2 * qvec[2] ** 2 - 2 * qvec[3] ** 2,
2 * qvec[1] * qvec[2] - 2 * qvec[0] * qvec[3],
2 * qvec[3] * qvec[1] + 2 * qvec[0] * qvec[2],
],
[
2 * qvec[1] * qvec[2] + 2 * qvec[0] * qvec[3],
1 - 2 * qvec[1] ** 2 - 2 * qvec[3] ** 2,
2 * qvec[2] * qvec[3] - 2 * qvec[0] * qvec[1],
],
[
2 * qvec[3] * qvec[1] - 2 * qvec[0] * qvec[2],
2 * qvec[2] * qvec[3] + 2 * qvec[0] * qvec[1],
1 - 2 * qvec[1] ** 2 - 2 * qvec[2] ** 2,
],
]
)
def rotmat2qvec(R):
"""Convert rotation matrix to quaternion.
Args:
R: Rotation matrix of shape (3, 3).
Returns:
Quaternion vector of shape (4,).
"""
rxx, ryx, rzx, rxy, ryy, rzy, rxz, ryz, rzz = R.flat
K = (
np.array(
[
[rxx - ryy - rzz, 0, 0, 0],
[ryx + rxy, ryy - rxx - rzz, 0, 0],
[rzx + rxz, rzy + ryz, rzz - rxx - ryy, 0],
[ryz - rzy, rzx - rxz, rxy - ryx, rxx + ryy + rzz],
]
)
/ 3.0
)
eigvals, eigvecs = np.linalg.eigh(K)
qvec = eigvecs[np.array([3, 0, 1, 2]), np.argmax(eigvals)]
if qvec[0] < 0:
qvec *= -1
return qvec
def get_vocab_tree() -> Path:
"""Return path to vocab tree. Downloads vocab tree if it doesn't exist.
@@ -582,40 +183,54 @@ def run_colmap(
def colmap_to_json(
cameras_path: Path,
images_path: Path,
recon_dir: Path,
output_dir: Path,
camera_model: CameraModel,
camera_mask_path: Optional[Path] = None,
image_id_to_depth_path: Optional[Dict[int, Path]] = None,
image_rename_map: Optional[Dict[str, str]] = None,
) -> int:
"""Converts COLMAP's cameras.bin and images.bin to a JSON file.
Args:
cameras_path: Path to the cameras.bin file.
images_path: Path to the images.bin file.
recon_dir: Path to the reconstruction directory, e.g. "sparse/0"
output_dir: Path to the output directory.
camera_mask_path: Path to the camera mask.
camera_model: Camera model used.
camera_mask_path: Path to the camera mask.
image_id_to_depth_path: When including sfm-based depth, embed these depth file paths in the exported json
image_rename_map: Use these image names instead of the names embedded in the COLMAP db
Returns:
The number of registered images.
"""
cameras = read_cameras_binary(cameras_path)
images = read_images_binary(images_path)
# TODO(1480) use pycolmap
# recon = pycolmap.Reconstruction(recon_dir)
# cam_id_to_camera = recon.cameras
# im_id_to_image = recon.images
cam_id_to_camera = read_cameras_binary(recon_dir / "cameras.bin")
im_id_to_image = read_images_binary(recon_dir / "images.bin")
# Only supports one camera
camera_params = cameras[1].params
# Only support first camera
CAMERA_ID = 1 # pylint: disable=invalid-name
camera_params = cam_id_to_camera[CAMERA_ID].params
frames = []
for _, im_data in images.items():
for im_id, im_data in im_id_to_image.items():
# NB: COLMAP uses Eigen / scalar-first quaternions
# * https://colmap.github.io/format.html
# * https://github.com/colmap/colmap/blob/bf3e19140f491c3042bfd85b7192ef7d249808ec/src/base/pose.cc#L75
# the `rotation_matrix()` handles that format for us.
# TODO(1480) BEGIN use pycolmap API
# rotation = im_data.rotation_matrix()
rotation = qvec2rotmat(im_data.qvec)
translation = im_data.tvec.reshape(3, 1)
w2c = np.concatenate([rotation, translation], 1)
w2c = np.concatenate([w2c, np.array([[0, 0, 0, 1]])], 0)
c2w = np.linalg.inv(w2c)
# Convert from COLMAP's camera coordinate system to ours
# Convert from COLMAP's camera coordinate system (OpenCV) to ours (OpenGL)
c2w[0:3, 1:3] *= -1
c2w = c2w[np.array([1, 0, 2, 3]), :]
c2w[2, :] *= -1
@@ -628,9 +243,13 @@ def colmap_to_json(
frame = {
"file_path": name.as_posix(),
"transform_matrix": c2w.tolist(),
"colmap_im_id": im_id,
}
if camera_mask_path is not None:
frame["mask_path"] = camera_mask_path.relative_to(camera_mask_path.parent.parent).as_posix()
if image_id_to_depth_path is not None:
depth_path = image_id_to_depth_path[im_id]
frame["depth_file_path"] = str(depth_path)
frames.append(frame)
out = {
@@ -638,8 +257,8 @@ def colmap_to_json(
"fl_y": float(camera_params[1]),
"cx": float(camera_params[2]),
"cy": float(camera_params[3]),
"w": cameras[1].width,
"h": cameras[1].height,
"w": cam_id_to_camera[CAMERA_ID].width,
"h": cam_id_to_camera[CAMERA_ID].height,
"camera_model": camera_model.value,
}
@@ -670,6 +289,151 @@ def colmap_to_json(
return len(frames)
def create_sfm_depth(
recon_dir: Path,
output_dir: Path,
verbose: bool = True,
depth_scale_to_integer_factor: float = 1000.0,
min_depth: float = 0.001,
max_depth: float = 10000,
max_repoj_err: float = 2.5,
min_n_visible: int = 2,
include_depth_debug: bool = False,
input_images_dir: Optional[Path] = None,
) -> Dict[int, Path]:
"""Converts COLMAP's points3d.bin to sparse depth map images encoded as
16-bit "millimeter depth" PNGs.
Notes:
* This facility does NOT use COLMAP dense reconstruction; it creates depth
maps from sparse SfM points here.
* COLMAP does *not* reconstruct metric depth unless you give it calibrated
(metric) intrinsics as input. Therefore, "depth" in this function has
potentially ambiguous units.
Args:
recon_dir: Path to the reconstruction directory, e.g. "sparse/0"
output_dir: Path to the output directory.
verbose: If True, logs progress of depth image creation.
depth_scale_to_integer_factor: Use this parameter to tune the conversion of
raw depth measurements to integer depth values. This value should
be equal to 1. / `depth_unit_scale_factor`, where
`depth_unit_scale_factor` is the value you provide at training time.
E.g. for millimeter depth, leave `depth_unit_scale_factor` at 1e-3
and depth_scale_to_integer_factor at 1000.
min_depth: Discard points closer than this to the camera.
max_depth: Discard points farther than this from the camera.
max_repoj_err: Discard points with reprojection error greater than this
amount (in pixels).
min_n_visible: Discard 3D points that have been triangulated with fewer
than this many frames.
include_depth_debug: Also include debug images showing depth overlaid
upon RGB.
Returns:
Depth file paths indexed by COLMAP image id
"""
# TODO(1480) use pycolmap
# recon = pycolmap.Reconstruction(recon_dir)
# ptid_to_info = recon.points3D
# cam_id_to_camera = recon.cameras
# im_id_to_image = recon.images
ptid_to_info = read_points3d_binary(recon_dir / "points3D.bin")
cam_id_to_camera = read_cameras_binary(recon_dir / "cameras.bin")
im_id_to_image = read_images_binary(recon_dir / "images.bin")
# Only support first camera
CAMERA_ID = 1 # pylint: disable=invalid-name
W = cam_id_to_camera[CAMERA_ID].width
H = cam_id_to_camera[CAMERA_ID].height
if verbose:
iter_images = track(
im_id_to_image.items(), total=len(im_id_to_image.items()), description="Creating depth maps ..."
)
else:
iter_images = iter(im_id_to_image.items())
image_id_to_depth_path = {}
for im_id, im_data in iter_images:
# TODO(1480) BEGIN delete when abandoning colmap_parsing_utils
pids = [pid for pid in im_data.point3D_ids if pid != -1]
xyz_world = np.array([ptid_to_info[pid].xyz for pid in pids])
rotation = qvec2rotmat(im_data.qvec)
z = (rotation @ xyz_world.T)[-1] + im_data.tvec[-1]
errors = np.array([ptid_to_info[pid].error for pid in pids])
n_visible = np.array([len(ptid_to_info[pid].image_ids) for pid in pids])
uv = np.array([im_data.xys[i] for i in range(len(im_data.xys)) if im_data.point3D_ids[i] != -1])
# TODO(1480) END delete when abandoning colmap_parsing_utils
# TODO(1480) BEGIN use pycolmap API
# # Get only keypoints that have corresponding triangulated 3D points
# p2ds = im_data.get_valid_points2D()
# xyz_world = np.array([ptid_to_info[p2d.point3D_id].xyz for p2d in p2ds])
# # COLMAP OpenCV convention: z is always positive
# z = (im_data.rotation_matrix() @ xyz_world.T)[-1] + im_data.tvec[-1]
# # Mean reprojection error in image space
# errors = np.array([ptid_to_info[p2d.point3D_id].error for p2d in p2ds])
# # Number of frames in which each frame is visible
# n_visible = np.array([ptid_to_info[p2d.point3D_id].track.length() for p2d in p2ds])
# Note: these are *unrectified* pixel coordinates that should match the original input
# no matter the camera model
# uv = np.array([p2d.xy for p2d in p2ds])
# TODO(1480) END use pycolmap API
idx = np.where(
(z >= min_depth)
& (z <= max_depth)
& (errors <= max_repoj_err)
& (n_visible >= min_n_visible)
& (uv[:, 0] >= 0)
& (uv[:, 0] < W)
& (uv[:, 1] >= 0)
& (uv[:, 1] < H)
)
z = z[idx]
uv = uv[idx]
uu, vv = uv[:, 0].astype(int), uv[:, 1].astype(int)
depth = np.zeros((H, W), dtype=np.float32)
depth[vv, uu] = z
# E.g. if `depth` is metric and in units of meters, and `depth_scale_to_integer_factor`
# is 1000, then `depth_img` will be integer millimeters.
depth_img = (depth_scale_to_integer_factor * depth).astype(np.uint16)
out_name = str(im_data.name)
depth_path = output_dir / out_name
cv2.imwrite(str(depth_path), depth_img)
image_id_to_depth_path[im_id] = depth_path
if include_depth_debug:
assert input_images_dir is not None, "Need explicit input_images_dir for debug images"
assert input_images_dir.exists(), input_images_dir
depth_flat = depth.flatten()[:, None]
overlay = 255.0 * colormaps.apply_depth_colormap(torch.from_numpy(depth_flat)).numpy()
overlay = overlay.reshape([H, W, 3])
input_image_path = input_images_dir / im_data.name
input_image = cv2.imread(str(input_image_path))
debug = 0.3 * input_image + 0.7 + overlay
out_name = out_name + ".debug.jpg"
output_path = output_dir / "debug_depth" / out_name
output_path.parent.mkdir(parents=True, exist_ok=True)
cv2.imwrite(str(output_path), debug.astype(np.uint8))
return image_id_to_depth_path
def get_matching_summary(num_intial_frames: int, num_matched_frames: int) -> str:
"""Returns a summary of the matching results.
+5 -2
View File
@@ -28,6 +28,7 @@ from typing_extensions import Literal
from nerfstudio.process_data.process_data_utils import CameraModel
try:
# TODO(1480) un-hide pycolmap import
import pycolmap
from hloc import (
extract_features,
@@ -95,14 +96,16 @@ def run_hloc(
pairs_from_retrieval.main(retrieval_path, sfm_pairs, num_matched=num_matched)
match_features.main(matcher_conf, sfm_pairs, features=features, matches=matches)
image_options = pycolmap.ImageReaderOptions(camera_model=camera_model.value)
image_options = pycolmap.ImageReaderOptions( # pylint: disable=c-extension-no-member
camera_model=camera_model.value
)
reconstruction.main(
sfm_dir,
image_dir,
sfm_pairs,
features,
matches,
camera_mode=pycolmap.CameraMode.SINGLE,
camera_mode=pycolmap.CameraMode.SINGLE, # pylint: disable=c-extension-no-member
image_options=image_options,
verbose=verbose,
)
+3 -1
View File
@@ -216,7 +216,9 @@ def process_depth_maps(
# Downscale depth maps
summary_log.append(
process_data_utils.downscale_images(depth_dir, num_downscales, folder_name="depth", verbose=verbose)
process_data_utils.downscale_images(
depth_dir, num_downscales, folder_name="depths", nearest_neighbor=True, verbose=verbose
)
)
return summary_log, polycam_depth_maps_filenames
+12 -2
View File
@@ -263,7 +263,13 @@ def copy_images(data: Path, image_dir: Path, verbose) -> OrderedDict[Path, Path]
return OrderedDict((original_path, new_path) for original_path, new_path in zip(image_paths, copied_images))
def downscale_images(image_dir: Path, num_downscales: int, folder_name: str = "images", verbose: bool = False) -> str:
def downscale_images(
image_dir: Path,
num_downscales: int,
folder_name: str = "images",
nearest_neighbor: bool = False,
verbose: bool = False,
) -> str:
"""Downscales the images in the directory. Uses FFMPEG.
Assumes images are named frame_00001.png, frame_00002.png, etc.
@@ -272,6 +278,7 @@ def downscale_images(image_dir: Path, num_downscales: int, folder_name: str = "i
image_dir: Path to the directory containing the images.
num_downscales: Number of times to downscale the images. Downscales by 2 each time.
folder_name: Name of the output folder
nearest_neighbor: Use nearest neighbor sampling (useful for depth images)
verbose: If True, logs the output of the command.
Returns:
@@ -291,10 +298,13 @@ def downscale_images(image_dir: Path, num_downscales: int, folder_name: str = "i
# Using %05d ffmpeg commands appears to be unreliable (skips images), so use scandir.
files = os.scandir(image_dir)
for f in files:
if f.is_dir():
continue
filename = f.name
nn_flag = "" if not nearest_neighbor else ":flags=neighbor"
ffmpeg_cmd = [
f'ffmpeg -y -noautorotate -i "{image_dir / filename}" ',
f"-q:v 2 -vf scale=iw/{downscale_factor}:ih/{downscale_factor} ",
f"-q:v 2 -vf scale=iw/{downscale_factor}:ih/{downscale_factor}{nn_flag} ",
f'"{downscale_dir / filename}"',
]
ffmpeg_cmd = " ".join(ffmpeg_cmd)
@@ -46,6 +46,8 @@ def record3d_to_json(images_paths: List[Path], metadata_path: Path, output_dir:
metadata_dict = io.load_from_json(metadata_path)
poses_data = np.array(metadata_dict["poses"]) # (N, 3, 4)
# NB: Record3D / scipy use "scalar-last" format quaternions (x y z w)
# https://fzheng.me/2017/11/12/quaternion_conventions_en/
camera_to_worlds = np.concatenate(
[Rotation.from_quat(poses_data[:, :4]).as_matrix(), poses_data[:, 4:, None]],
axis=-1,
+2 -2
View File
@@ -31,7 +31,7 @@ def apply_colormap(image: TensorType["bs":..., 1], cmap="viridis") -> TensorType
cmap: Colormap for image.
Returns:
TensorType: Colored image
TensorType: Colored image with colors in [0, 1]
"""
colormap = cm.get_cmap(cmap)
@@ -62,7 +62,7 @@ def apply_depth_colormap(
cmap: Colormap to apply.
Returns:
Colored depth image
Colored depth image with colors in [0, 1]
"""
near_plane = near_plane or float(torch.min(depth))
+8
View File
@@ -16,6 +16,7 @@
Miscellaneous helper code.
"""
from typing import Any, Callable, Dict, List, Optional, Union
import torch
@@ -147,3 +148,10 @@ def update_avg(prev_avg: float, new_val: float, step: int) -> float:
float: new updated average
"""
return (step * prev_avg + new_val) / (step + 1)
def strtobool(val) -> bool:
"""Cheap replacement for `distutils.util.strtobool()` which is deprecated
FMI https://stackoverflow.com/a/715468
"""
return val.lower() in ("yes", "y", "true", "t", "on", "1")
+3
View File
@@ -38,10 +38,13 @@ dependencies = [
"Pillow>=9.3.0",
"plotly>=5.7.0",
"protobuf<=3.20.3,!=3.20.0",
# TODO(1480) enable when pycolmap windows wheels are available
# "pycolmap==0.3.0",
"pyequilib>=0.5.6",
"pymeshlab>=2022.2.post2",
"pyngrok>=5.1.0",
"python-socketio>=5.7.1",
"pyquaternion>=0.9.9",
"requests",
"rich>=12.5.1",
"scikit-image>=0.19.3",
+77 -8
View File
@@ -91,10 +91,14 @@ class ProcessImages:
"""
gpu: bool = True
"""If True, use GPU."""
use_sfm_depth: bool = False
"""If True, export and use depth maps induced from SfM points."""
include_depth_debug: bool = False
"""If --use-sfm-depth and this flag is True, also export debug images showing SfM overlaid upon input images."""
verbose: bool = False
"""If True, print extra logging."""
def main(self) -> None:
def main(self) -> None: # pylint: disable=R0915
"""Process images into a nerfstudio dataset."""
# pylint: disable=too-many-statements
require_cameras_exist = False
@@ -161,14 +165,33 @@ class ProcessImages:
# Colmap uses renamed images
image_rename_map = None
# Export depth maps
if self.use_sfm_depth:
depth_dir = self.output_dir / "depth"
depth_dir.mkdir(parents=True, exist_ok=True)
image_id_to_depth_path = colmap_utils.create_sfm_depth(
recon_dir=colmap_dir / "sparse" / "0",
output_dir=depth_dir,
include_depth_debug=self.include_depth_debug,
input_images_dir=image_dir,
verbose=self.verbose,
)
summary_log.append(
process_data_utils.downscale_images(
depth_dir, self.num_downscales, folder_name="depths", nearest_neighbor=True, verbose=self.verbose
)
)
else:
image_id_to_depth_path = {}
# Save transforms.json
if (colmap_model_path / "cameras.bin").exists():
with CONSOLE.status("[bold yellow]Saving results to transforms.json", spinner="balloon"):
num_matched_frames = colmap_utils.colmap_to_json(
cameras_path=colmap_model_path / "cameras.bin",
images_path=colmap_model_path / "images.bin",
recon_dir=colmap_dir / "sparse" / "0",
output_dir=self.output_dir,
camera_model=CAMERA_MODELS[self.camera_type],
image_id_to_depth_path=image_id_to_depth_path,
image_rename_map=image_rename_map,
)
summary_log.append(f"Colmap matched {num_matched_frames} images")
@@ -272,12 +295,16 @@ class ProcessVideo:
"""Create circle crop mask. The radius is the percent of the image diagonal."""
percent_crop: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0)
"""Percent of the image to crop. (top, bottom, left, right)"""
use_sfm_depth: bool = False
"""If True, export and use depth maps induced from SfM points."""
include_depth_debug: bool = False
"""If --use-sfm-depth and this flag is True, also export debug images showing SfM overlaid upon input images."""
gpu: bool = True
"""If True, use GPU."""
verbose: bool = False
"""If True, print extra logging."""
def main(self) -> None:
def main(self) -> None: # pylint: disable=R0915
"""Process video into a nerfstudio dataset."""
install_checks.check_ffmpeg_installed()
install_checks.check_colmap_installed()
@@ -372,14 +399,33 @@ class ProcessVideo:
CONSOLE.log("[bold red]Invalid combination of sfm_tool, feature_type, and matcher_type, exiting")
sys.exit(1)
# Export depth maps
if self.use_sfm_depth:
depth_dir = self.output_dir / "depth"
depth_dir.mkdir(parents=True, exist_ok=True)
image_id_to_depth_path = colmap_utils.create_sfm_depth(
recon_dir=colmap_dir / "sparse" / "0",
output_dir=depth_dir,
include_depth_debug=self.include_depth_debug,
input_images_dir=image_dir,
verbose=self.verbose,
)
summary_log.append(
process_data_utils.downscale_images(
depth_dir, self.num_downscales, folder_name="depths", nearest_neighbor=True, verbose=self.verbose
)
)
else:
image_id_to_depth_path = {}
# Save transforms.json
if (colmap_dir / "sparse" / "0" / "cameras.bin").exists():
with CONSOLE.status("[bold yellow]Saving results to transforms.json", spinner="balloon"):
num_matched_frames = colmap_utils.colmap_to_json(
cameras_path=colmap_dir / "sparse" / "0" / "cameras.bin",
images_path=colmap_dir / "sparse" / "0" / "images.bin",
recon_dir=colmap_dir / "sparse" / "0",
output_dir=self.output_dir,
camera_model=CAMERA_MODELS[self.camera_type],
image_id_to_depth_path=image_id_to_depth_path,
camera_mask_path=mask_path,
image_rename_map=None,
)
@@ -426,6 +472,10 @@ class ProcessInsta360:
"""If True, skips COLMAP and generates transforms.json if possible."""
colmap_cmd: str = "colmap"
"""How to call the COLMAP executable."""
use_sfm_depth: bool = False
"""If True, export and use depth maps induced from SfM points."""
include_depth_debug: bool = False
"""If --use-sfm-depth and this flag is True, also export debug images showing SfM overlaid upon input images."""
gpu: bool = True
"""If True, use GPU."""
verbose: bool = False
@@ -493,14 +543,33 @@ class ProcessInsta360:
colmap_cmd=self.colmap_cmd,
)
# Export depth maps
if self.use_sfm_depth:
depth_dir = self.output_dir / "depth"
depth_dir.mkdir(parents=True, exist_ok=True)
image_id_to_depth_path = colmap_utils.create_sfm_depth(
recon_dir=colmap_dir / "sparse" / "0",
output_dir=depth_dir,
include_depth_debug=self.include_depth_debug,
input_images_dir=image_dir,
verbose=self.verbose,
)
summary_log.append(
process_data_utils.downscale_images(
depth_dir, self.num_downscales, folder_name="depths", nearest_neighbor=True, verbose=self.verbose
)
)
else:
image_id_to_depth_path = {}
# Save transforms.json
if (colmap_dir / "sparse" / "0" / "cameras.bin").exists():
with CONSOLE.status("[bold yellow]Saving results to transforms.json", spinner="balloon"):
num_matched_frames = colmap_utils.colmap_to_json(
cameras_path=colmap_dir / "sparse" / "0" / "cameras.bin",
images_path=colmap_dir / "sparse" / "0" / "images.bin",
recon_dir=colmap_dir / "sparse" / "0",
output_dir=self.output_dir,
camera_model=CAMERA_MODELS["fisheye"],
image_id_to_depth_path=image_id_to_depth_path,
image_rename_map=None,
)
summary_log.append(f"Colmap matched {num_matched_frames} images")
+63
View File
@@ -0,0 +1,63 @@
"""
Test misc data utils
"""
import numpy as np
from pyquaternion import Quaternion
from scipy.spatial.transform import Rotation
# TODO(1480) use pycolmap instead of colmap_parsing_utils
# import pycolmap
from nerfstudio.data.utils.colmap_parsing_utils import qvec2rotmat
def test_scalar_first_scalar_last_quaternions():
"""Many nerfstudio datasets use quaternions for pose rotations. Some use
scalar-first format and others use scalar-last. However internally
nerfstudio uses rotation *matrices* / projection matrices. This test
helps document differences in quaternion encodings & conversion to
rotation matrices.
FMI see call sites in nerfstudio where the various libraries are used. You
might also consult this detailed reference:
* "Why and How to Avoid the Flipped Quaternion Multiplication"
Sommer et al.
https://arxiv.org/abs/1801.07478
"""
# Pick an arbitrary euler rotation; note that different communities
# also use different rotation axes (e.g. Euler vs Tait-Bryan ...)
rot = Rotation.from_euler("xyz", np.array([10.0, 20.0, 30.0]), degrees=True)
# scalar-last
xyzw = rot.as_quat()
assert np.allclose(xyzw, np.array([0.03813458, 0.18930786, 0.23929834, 0.95154852]))
# scalar-first
wxyz = xyzw[[3, 0, 1, 2]]
assert np.allclose(wxyz, np.array([0.95154852, 0.03813458, 0.18930786, 0.23929834]))
# Expected Rotation matrix
# fmt: off
R_expected = np.array( # pylint: disable=invalid-name
[
[ 0.81379768, -0.44096961, 0.37852231],
[ 0.46984631, 0.88256412, 0.01802831],
[-0.34202014, 0.16317591, 0.92541658]
]
)
# fmt: on
# Record3D / scipy
R = Rotation.from_quat(xyzw).as_matrix()
assert np.allclose(R, R_expected)
# Nuscenes / pyquaternion
R = Quaternion(wxyz).rotation_matrix
assert np.allclose(R, R_expected)
# COLMAP
# TODO(1480) use pycolmap
# R = pycolmap.qvec_to_rotmat(wxyz)
R = qvec2rotmat(wxyz)
assert np.allclose(R, R_expected)
+1 -1
View File
@@ -5,11 +5,11 @@ import importlib
import math
import os
import random
from distutils.util import strtobool
import torch
from nerfstudio.utils.math import intersect_aabb
from nerfstudio.utils.misc import strtobool
test_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")