This commit is contained in:
Justin committed 2023-03-23 05:38:00 +00:00
commit 03a4a45e8f
16 files changed
+243 -74

No files matched your search

+9
View File
@@ -23,11 +23,14 @@ from dataclasses import dataclass
from pathlib import Path
from typing import Any, List, Optional, Tuple, Type
from typing_extensions import Literal
# model instances
from nerfstudio.utils import writer
warnings.filterwarnings("ignore", module="torchvision")
# Pretty printing class
class PrintableConfig: # pylint: disable=too-few-public-methods
"""Printable Config defining str function"""
@@ -148,3 +151,9 @@ class ViewerConfig(PrintableConfig):
actually used in training/evaluation. If -1, display all."""
quit_on_train_completion: bool = False
"""Whether to kill the training job when it has completed. Note this will stop rendering in the viewer."""
image_format: Literal["jpeg", "png"] = "jpeg"
"""Image format viewer should use; jpeg is lossy compression, while png is lossless."""
jpeg_quality: int = 90
"""Quality tradeoff to use for jpeg compression."""
png_compression: int = 1
"""Size/speed tradeoff to use for png compression."""
@@ -24,6 +24,7 @@ from typing import Any, Dict, List, Optional, Type
import torch
from torchtyping import TensorType
from typing_extensions import Literal
import nerfstudio.configs.base_config as cfg
from nerfstudio.cameras.cameras import Cameras
@@ -89,6 +90,26 @@ class DataparserOutputs:
with open(path, "w", encoding="UTF-8") as file:
json.dump(data, file, indent=4)
def transform_poses_to_original_space(
self,
poses: TensorType["num_poses", 3, 4],
camera_convention: Literal["opengl", "opencv"] = "opencv",
) -> TensorType["num_poses", 3, 4]:
"""
Transforms the poses in the transformed space back to the original world coordinate system.
Args:
poses: Poses in the transformed space
camera_convention: Camera system convention used for the transformed poses
Returns:
Original poses
"""
return transform_poses_to_original_space(
poses,
self.dataparser_transform,
self.dataparser_scale,
camera_convention=camera_convention,
)
@dataclass
class DataParserConfig(cfg.InstantiateConfig):
@@ -136,3 +157,46 @@ class DataParser:
"""
dataparser_outputs = self._generate_dataparser_outputs(split)
return dataparser_outputs
def transform_poses_to_original_space(
poses: TensorType["num_poses", 3, 4],
applied_transform: TensorType[3, 4],
applied_scale: float,
camera_convention: Literal["opengl", "opencv"] = "opencv",
) -> TensorType["num_poses", 3, 4]:
"""
Transforms the poses in the transformed space back to the original world coordinate system.
Args:
poses: Poses in the transformed space
applied_transform: Transform matrix applied in the data processing step
applied_scale: Scale used in the data processing step
camera_convention: Camera system convention used for the transformed poses
Returns:
Original poses
"""
output_poses = torch.cat(
(
poses,
torch.tensor([[[0, 0, 0, 1]]], dtype=poses.dtype, device=poses.device).repeat_interleave(len(poses), 0),
),
1,
)
output_poses[..., :3, 3] /= applied_scale
inv_transform = torch.linalg.inv(
torch.cat(
(
applied_transform,
torch.tensor([[0, 0, 0, 1]], dtype=applied_transform.dtype, device=applied_transform.device),
),
0,
)
)
output_poses = torch.einsum("ij,bjk->bik", inv_transform, output_poses)
if camera_convention == "opencv":
output_poses[..., 0:3, 1:3] *= -1
elif camera_convention == "opengl":
pass
else:
raise ValueError(f"Camera convention {camera_convention} is not supported.")
return output_poses[:, :3]
@@ -78,18 +78,19 @@ def _parse_osm_txt(filename: str):
return np.array([float(x) for x in nums]).reshape([4, 4]).astype(np.float32)
def get_camera_params(scene_dir: str, split: str) -> Tuple[torch.Tensor, torch.Tensor, int]:
def get_camera_params(
scene_dir: str, split: Literal["train", "validation", "test"]
) -> Tuple[torch.Tensor, torch.Tensor, int]:
"""Load camera intrinsic and extrinsic parameters for a given scene split.
Args"
scene_dir : The directory containing the scene data.
split : The split for which to load the camera parameters. Either "train", "val", or "test".
split : The split for which to load the camera parameters.
Returns
A tuple containing the intrinsic parameters (as a torch.Tensor of shape [N, 4, 4]),
the camera-to-world matrices (as a torch.Tensor of shape [N, 4, 4]), and the number of cameras (N).
"""
split = "validation" if split == "val" else split
split_dir = f"{scene_dir}/{split}"
# camera parameters files
@@ -169,7 +170,7 @@ class NeRFOSR(DataParser):
# get all split cam params
intrinsics_train, camera_to_worlds_train, n_train = get_camera_params(scene_dir, "train")
intrinsics_val, camera_to_worlds_val, n_val = get_camera_params(scene_dir, "val")
intrinsics_val, camera_to_worlds_val, n_val = get_camera_params(scene_dir, "validation")
intrinsics_test, camera_to_worlds_test, _ = get_camera_params(scene_dir, "test")
# combine all cam params
@@ -192,7 +193,7 @@ class NeRFOSR(DataParser):
if split == "train":
camera_to_worlds = camera_to_worlds[:n_train]
intrinsics = intrinsics[:n_train]
elif split == "val":
elif split == "validation":
camera_to_worlds = camera_to_worlds[n_train : n_train + n_val]
intrinsics = intrinsics[n_train : n_train + n_val]
elif split == "test":
+12 -11
View File
@@ -138,6 +138,18 @@ class Trainer:
'test': loads train/test datasets into memory
'inference': does not load any dataset into memory
"""
# set up viewer if enabled
viewer_log_path = self.base_dir / self.config.viewer.relative_log_filename
self.viewer_state, banner_messages = None, None
if self.config.is_viewer_enabled() and self.local_rank == 0:
datapath = self.config.data
if datapath is None:
datapath = self.base_dir
self.viewer_state, banner_messages = viewer_utils.setup_viewer(
self.config.viewer, log_filename=viewer_log_path, datapath=datapath
)
self._check_viewer_warnings()
self.pipeline = self.config.pipeline.setup(
device=self.device, test_mode=test_mode, world_size=self.world_size, local_rank=self.local_rank
)
@@ -153,17 +165,6 @@ class Trainer:
)
)
# set up viewer if enabled
viewer_log_path = self.base_dir / self.config.viewer.relative_log_filename
self.viewer_state, banner_messages = None, None
if self.config.is_viewer_enabled() and self.local_rank == 0:
datapath = self.pipeline.datamanager.get_datapath()
if datapath is None:
datapath = self.base_dir
self.viewer_state, banner_messages = viewer_utils.setup_viewer(
self.config.viewer, log_filename=viewer_log_path, datapath=datapath
)
self._check_viewer_warnings()
# set up writers/profilers if enabled
writer_log_path = self.base_dir / self.config.logging.relative_log_dir
writer.setup_event_writer(
+1 -1
View File
@@ -338,7 +338,7 @@ class MiDaSMSELoss(nn.Module):
def __init__(self, reduction_type: Literal["image", "batch"] = "batch"):
super().__init__()
self.reduction_type = reduction_type
self.reduction_type: Literal["image", "batch"] = reduction_type
# reduction here is different from the image/batch-based reduction. This is either "mean" or "sum"
self.mse_loss = MSELoss(reduction="none")
@@ -641,6 +641,7 @@ class NeuSSampler(Sampler):
# Start with uniform sampling
if ray_samples is None:
ray_samples = self.uniform_sampler(ray_bundle, num_samples=self.num_samples)
assert ray_samples is not None
total_iters = 0
sorted_index = None
@@ -697,6 +698,7 @@ class NeuSSampler(Sampler):
"""
batch_size = ray_samples.shape[0]
prev_sdf, next_sdf = sdf[:, :-1], sdf[:, 1:]
assert ray_samples.deltas is not None
deltas = ray_samples.deltas[:, :-1, 0]
mid_sdf = (prev_sdf + next_sdf) * 0.5
cos_val = (next_sdf - prev_sdf) / (deltas + 1e-5)
@@ -738,6 +740,9 @@ class NeuSSampler(Sampler):
ray_samples_2 : ray_samples to merge
"""
assert ray_samples_1.spacing_starts is not None and ray_samples_2.spacing_starts is not None
assert ray_samples_1.spacing_ends is not None and ray_samples_2.spacing_ends is not None
assert ray_samples_1.spacing_to_euclidean_fn is not None
starts_1 = ray_samples_1.spacing_starts[..., 0]
starts_2 = ray_samples_2.spacing_starts[..., 0]
+1
View File
@@ -101,6 +101,7 @@ class Model(nn.Module):
# NOTE: call `super().populate_modules()` in subclasses
if self.config.enable_collider:
assert self.config.collider_params is not None
self.collider = NearFarCollider(
near_plane=self.config.collider_params["near_plane"], far_plane=self.config.collider_params["far_plane"]
)
+3
View File
@@ -90,6 +90,8 @@ class TensoRFModel(Model):
config: TensoRF configuration to instantiate model
"""
config: TensoRFModelConfig
def __init__(
self,
config: TensoRFModelConfig,
@@ -303,6 +305,7 @@ class TensoRFModel(Model):
image = batch["image"].to(outputs["rgb"].device)
rgb = outputs["rgb"]
acc = colormaps.apply_colormap(outputs["accumulation"])
assert self.config.collider_params is not None
depth = colormaps.apply_depth_colormap(
outputs["depth"],
accumulation=outputs["accumulation"],
+1
View File
@@ -201,6 +201,7 @@ class NeRFModel(Model):
rgb_fine = outputs["rgb_fine"]
acc_coarse = colormaps.apply_colormap(outputs["accumulation_coarse"])
acc_fine = colormaps.apply_colormap(outputs["accumulation_fine"])
assert self.config.collider_params is not None
depth_coarse = colormaps.apply_depth_colormap(
outputs["depth_coarse"],
accumulation=outputs["accumulation_coarse"],
+2
View File
@@ -95,6 +95,7 @@ class Pipeline(nn.Module):
datamanager: DataManager
_model: Model
world_size: int
@property
def model(self):
@@ -201,6 +202,7 @@ class VanillaPipelineConfig(cfg.InstantiateConfig):
class VanillaPipeline(Pipeline):
"""The pipeline class for the vanilla nerf setup of multiple cameras for one or a few scenes.
Args:
config: configuration to instantiate pipeline
device: location to place model and data
test_mode:
+5
View File
@@ -75,6 +75,11 @@ def run_hloc(
camera_model: Camera model to use.
gpu: If True, use GPU.
verbose: If True, logs the output of the command.
matching_method: Method to use for matching images.
feature_type: Type of visual features to use.
matcher_type: Type of feature matcher to use.
num_matched: Number of image pairs for loc.
refine_pixsfm: If True, refine the reconstruction using pixel-perfect-sfm.
"""
if not _HAS_HLOC:
CONSOLE.print(
+77 -37
View File
@@ -27,7 +27,7 @@ from nerfstudio.process_data.process_data_utils import CAMERA_MODELS
CONSOLE = Console(width=120)
def _find_distortion_param(calib_xml: ET.Element, param_name: str):
def _find_param(calib_xml: ET.Element, param_name: str):
param = calib_xml.find(param_name)
if param is not None:
return float(param.text) # type: ignore
@@ -62,34 +62,71 @@ def metashape_to_json( # pylint: disable=too-many-statements
raise ValueError("No sensors found")
calibrated_sensors = [sensor for sensor in sensors if sensor.find("calibration")]
if len(calibrated_sensors) != 1:
raise ValueError("Only one sensor is supported for now")
sensor = calibrated_sensors[0]
if not calibrated_sensors:
raise ValueError("No calibrated sensor found in Metashape XML")
sensor_type = [s.get("type") for s in calibrated_sensors]
if sensor_type.count(sensor_type[0]) != len(sensor_type):
raise ValueError("All Metashape sensors do not have the same sensor type")
data = {}
if sensor_type[0] == "frame":
data["camera_model"] = CAMERA_MODELS["perspective"].value
elif sensor_type[0] == "fisheye":
data["camera_model"] = CAMERA_MODELS["fisheye"].value
elif sensor_type[0] == "spherical":
data["camera_model"] = CAMERA_MODELS["equirectangular"].value
else:
# Cylindrical and RPC sensor types are not supported
raise ValueError(f"Unsupported Metashape sensor type '{sensor_type[0]}'")
assert sensor is not None, "Sensor not found in Metashape XML"
resolution = sensor.find("resolution")
assert resolution is not None, "Resolution not found in Metashape xml"
data["w"] = int(resolution.get("width")) # type: ignore
data["h"] = int(resolution.get("height")) # type: ignore
sensor_dict = {}
for sensor in calibrated_sensors:
s = {}
resolution = sensor.find("resolution")
assert resolution is not None, "Resolution not found in Metashape xml"
s["w"] = int(resolution.get("width")) # type: ignore
s["h"] = int(resolution.get("height")) # type: ignore
calib = sensor.find("calibration")
assert calib is not None, "Calibration not found in Metashape xml"
data["fl_x"] = float(calib.find("f").text) # type: ignore
data["fl_y"] = float(calib.find("f").text) # type: ignore
data["cx"] = float(calib.find("cx").text) + data["w"] / 2.0 # type: ignore
data["cy"] = float(calib.find("cy").text) + data["h"] / 2.0 # type: ignore
calib = sensor.find("calibration")
f = calib.find("f")
if f is not None:
s["fl_x"] = s["fl_y"] = float(f.text) # type: ignore
s["cx"] = _find_param(calib, "cx") + s["w"] / 2.0 # type: ignore
s["cy"] = _find_param(calib, "cy") + s["h"] / 2.0 # type: ignore
data["k1"] = _find_distortion_param(calib, "k1")
data["k2"] = _find_distortion_param(calib, "k2")
data["k3"] = _find_distortion_param(calib, "k3")
data["k4"] = _find_distortion_param(calib, "k4")
data["p1"] = _find_distortion_param(calib, "p1")
data["p2"] = _find_distortion_param(calib, "p2")
s["k1"] = _find_param(calib, "k1")
s["k2"] = _find_param(calib, "k2")
s["k3"] = _find_param(calib, "k3")
s["k4"] = _find_param(calib, "k4")
s["p1"] = _find_param(calib, "p1")
s["p2"] = _find_param(calib, "p2")
data["camera_model"] = CAMERA_MODELS["perspective"].value
sensor_dict[sensor.get("id")] = s
components = chunk.find("components")
component_dict = {}
for component in components:
transform = component.find("transform")
if transform is not None:
rotation = transform.find("rotation")
if rotation is None:
r = np.eye(3)
else:
r = np.array([float(x) for x in rotation.text.split()]).reshape((3, 3))
translation = transform.find("translation")
if translation is None:
t = np.zeros(3)
else:
t = np.array([float(x) for x in translation.text.split()])
scale = transform.find("scale")
if scale is None:
s = 1.0
else:
s = float(scale.text)
m = np.eye(4)
m[:3, :3] = r
m[:3, 3] = t / s
component_dict[component.get("id")] = m
frames = []
cameras = chunk.find("cameras")
@@ -97,33 +134,36 @@ def metashape_to_json( # pylint: disable=too-many-statements
num_skipped = 0
for camera in cameras:
frame = {}
# Labels sometimes have a file extension. We remove it for consistency.
camera_label = camera.get("label").split(".")[0] # type: ignore
camera_label = camera.get("label")
if camera_label not in image_filename_map:
continue
# Labels sometimes have a file extension. Try without the extension.
# (maybe it's just a '.' in the image name)
camera_label = camera_label.split(".")[0] # type: ignore
if camera_label not in image_filename_map:
continue
frame["file_path"] = image_filename_map[camera_label].as_posix()
if camera.get("sensor_id") != sensor.get("id"):
sensor_id = camera.get("sensor_id")
if sensor_id not in sensor_dict:
# this should only happen when we have a sensor that doesn't have calibration
if verbose:
CONSOLE.print(f"Missing sensor calibration for {camera.get('label')}, Skipping")
num_skipped += 1
continue
# Add all sensor parameters to this frame.
frame.update(sensor_dict[sensor_id])
if camera.find("transform") is None:
if verbose:
CONSOLE.print(f"Missing transforms data for {camera.get('label')}, Skipping")
num_skipped += 1
continue
t = [float(x) for x in camera.find("transform").text.split()] # type: ignore
transform = np.array(
[
[t[8], -t[9], -t[10], t[11]],
[t[0], -t[1], -t[2], t[3]],
[t[4], -t[5], -t[6], t[7]],
[t[12], -t[13], -t[14], t[15]],
]
)
transform = np.array([float(x) for x in camera.find("transform").text.split()]).reshape((4, 4))
component_id = camera.get("component_id")
if component_id in component_dict:
transform = component_dict[component_id] @ transform
transform = transform[[2, 0, 1, 3], :]
transform[:, 1:3] *= -1
frame["transform_matrix"] = transform.tolist()
frames.append(frame)
@@ -159,7 +159,9 @@ def convert_video_to_images(
ffmpeg_cmd += f" -vf thumbnail={spacing},setpts=N/TB{crop_cmd} -r 1"
else:
CONSOLE.print("[bold red]Can't satisfy requested number of frames. Extracting all frames.")
ffmpeg_cmd += f" -pix_fmt bgr8 -vf {crop_cmd[1:]}"
ffmpeg_cmd += " -pix_fmt bgr8"
if crop_cmd != "":
ffmpeg_cmd += f" -vf {crop_cmd[1:]}"
ffmpeg_cmd += f" {out_filename}"
run_command(ffmpeg_cmd, verbose=verbose)
@@ -60,7 +60,7 @@ def realitycapture_to_json(
cameras.setdefault(column, []).append(value)
for name in cameras["#name"]:
camera_label = name.split(".")[0]
camera_label = name.rpartition(".")[0]
if camera_label in image_filename_map:
img = np.array(Image.open(output_dir / image_filename_map[camera_label]))
break
@@ -73,7 +73,7 @@ def realitycapture_to_json(
missing_image_data = 0
for i, name in enumerate(cameras["#name"]):
basename = name.split(".")[0]
basename = name.rpartition(".")[0]
if basename not in image_filename_map:
if verbose:
CONSOLE.print(f"Missing image for camera data {basename}, Skipping")
+11 -2
View File
@@ -670,8 +670,17 @@ class ViewerState:
image = selected_output[..., [2, 1, 0]].cpu().numpy()
data = cv2.imencode(".jpg", image, [cv2.IMWRITE_JPEG_QUALITY, 75])[1].tobytes()
data = str("data:image/jpeg;base64," + base64.b64encode(data).decode("ascii"))
data = cv2.imencode(
f".{self.config.image_format}",
image,
[
cv2.IMWRITE_JPEG_QUALITY,
self.config.jpeg_quality,
cv2.IMWRITE_PNG_COMPRESSION,
self.config.png_compression,
],
)[1].tobytes()
data = str(f"data:image/{self.config.image_format};base64," + base64.b64encode(data).decode("ascii"))
self.vis["render_img"].write(data)
def _update_viewer_stats(self, render_time: float, num_rays: int, image_height: int, image_width: int) -> None:
+41 -15
View File
@@ -12,12 +12,30 @@ from nerfstudio.data.dataparsers.nerfstudio_dataparser import NerfstudioDataPars
from nerfstudio.data.utils.colmap_parsing_utils import Camera
from nerfstudio.data.utils.colmap_parsing_utils import Image as ColmapImage
from nerfstudio.data.utils.colmap_parsing_utils import (
qvec2rotmat,
write_cameras_binary,
write_images_binary,
)
from scripts.process_data import ProcessImages
def random_quaternion(num_poses: int):
"""
Generates random rotation quatenion.
"""
u, v, w = np.random.uniform(size=(3, num_poses))
quaternion = np.stack(
(
np.sqrt(1 - u) * np.sin(2 * np.pi * v),
np.sqrt(1 - u) * np.cos(2 * np.pi * v),
np.sqrt(u) * np.sin(2 * np.pi * w),
np.sqrt(u) * np.cos(2 * np.pi * w),
),
-1,
)
return quaternion
def test_process_images_skip_colmap(tmp_path: Path):
"""
Test ns-process-data images
@@ -34,21 +52,21 @@ def test_process_images_skip_colmap(tmp_path: Path):
)
frames = {}
num_frames = 10
qvecs = np.random.uniform(size=(num_frames, 4))
qvecs = random_quaternion(num_frames)
tvecs = np.random.uniform(size=(num_frames, 3))
# original_poses = np.concatenate(
# (
# np.concatenate(
# (
# np.stack(list(map(qvec2rotmat, qvecs))),
# tvecs[:, :, None],
# ),
# -1,
# ),
# np.array([[[0, 0, 0, 1]]], dtype=qvecs.dtype).repeat(num_frames, 0),
# ),
# -2,
# )
original_poses = np.concatenate(
(
np.concatenate(
(
np.stack(list(map(qvec2rotmat, qvecs))),
tvecs[:, :, None],
),
-1,
),
np.array([[[0, 0, 0, 1]]], dtype=qvecs.dtype).repeat(num_frames, 0),
),
-2,
)
for i in range(num_frames):
frames[i + 1] = ColmapImage(i + 1, qvecs[i], tvecs[i], 1, f"image_{i}.png", [], [])
Image.new("RGB", (width, height)).save(tmp_path / "images" / f"image_{i}.png")
@@ -77,4 +95,12 @@ def test_process_images_skip_colmap(tmp_path: Path):
outputs = parser.get_dataparser_outputs("train")
assert len(outputs.image_filenames) == 9
assert torch.is_tensor(outputs.dataparser_transform)
# TODO @jkulhanek: Add tests if the loaded poses are the same
# Test if the original poses can be obtained back
gt_poses = original_poses[[0, 1, 2, 3, 4, 5, 6, 7, 9]]
dataparser_poses = outputs.transform_poses_to_original_space(outputs.cameras.camera_to_worlds, "opencv").numpy()
dataparser_poses = np.concatenate(
(dataparser_poses, np.array([[[0, 0, 0, 1]]]).repeat(len(dataparser_poses), 0)), 1
)
dataparser_poses = np.linalg.inv(dataparser_poses)
np.testing.assert_allclose(gt_poses, dataparser_poses, rtol=0, atol=1e-5)