mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-22 23:55:36 +08:00
Merge branch 'main' of https://github.com/nerfstudio-project/nerfstudio
This commit is contained in:
commit
03a4a45e8f
16 files changed
+243
-74
No files matched your search
@@ -23,11 +23,14 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, List, Optional, Tuple, Type
|
||||
|
||||
from typing_extensions import Literal
|
||||
|
||||
# model instances
|
||||
from nerfstudio.utils import writer
|
||||
|
||||
warnings.filterwarnings("ignore", module="torchvision")
|
||||
|
||||
|
||||
# Pretty printing class
|
||||
class PrintableConfig: # pylint: disable=too-few-public-methods
|
||||
"""Printable Config defining str function"""
|
||||
@@ -148,3 +151,9 @@ class ViewerConfig(PrintableConfig):
|
||||
actually used in training/evaluation. If -1, display all."""
|
||||
quit_on_train_completion: bool = False
|
||||
"""Whether to kill the training job when it has completed. Note this will stop rendering in the viewer."""
|
||||
image_format: Literal["jpeg", "png"] = "jpeg"
|
||||
"""Image format viewer should use; jpeg is lossy compression, while png is lossless."""
|
||||
jpeg_quality: int = 90
|
||||
"""Quality tradeoff to use for jpeg compression."""
|
||||
png_compression: int = 1
|
||||
"""Size/speed tradeoff to use for png compression."""
|
||||
@@ -24,6 +24,7 @@ from typing import Any, Dict, List, Optional, Type
|
||||
|
||||
import torch
|
||||
from torchtyping import TensorType
|
||||
from typing_extensions import Literal
|
||||
|
||||
import nerfstudio.configs.base_config as cfg
|
||||
from nerfstudio.cameras.cameras import Cameras
|
||||
@@ -89,6 +90,26 @@ class DataparserOutputs:
|
||||
with open(path, "w", encoding="UTF-8") as file:
|
||||
json.dump(data, file, indent=4)
|
||||
|
||||
def transform_poses_to_original_space(
|
||||
self,
|
||||
poses: TensorType["num_poses", 3, 4],
|
||||
camera_convention: Literal["opengl", "opencv"] = "opencv",
|
||||
) -> TensorType["num_poses", 3, 4]:
|
||||
"""
|
||||
Transforms the poses in the transformed space back to the original world coordinate system.
|
||||
Args:
|
||||
poses: Poses in the transformed space
|
||||
camera_convention: Camera system convention used for the transformed poses
|
||||
Returns:
|
||||
Original poses
|
||||
"""
|
||||
return transform_poses_to_original_space(
|
||||
poses,
|
||||
self.dataparser_transform,
|
||||
self.dataparser_scale,
|
||||
camera_convention=camera_convention,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DataParserConfig(cfg.InstantiateConfig):
|
||||
@@ -136,3 +157,46 @@ class DataParser:
|
||||
"""
|
||||
dataparser_outputs = self._generate_dataparser_outputs(split)
|
||||
return dataparser_outputs
|
||||
|
||||
|
||||
def transform_poses_to_original_space(
|
||||
poses: TensorType["num_poses", 3, 4],
|
||||
applied_transform: TensorType[3, 4],
|
||||
applied_scale: float,
|
||||
camera_convention: Literal["opengl", "opencv"] = "opencv",
|
||||
) -> TensorType["num_poses", 3, 4]:
|
||||
"""
|
||||
Transforms the poses in the transformed space back to the original world coordinate system.
|
||||
Args:
|
||||
poses: Poses in the transformed space
|
||||
applied_transform: Transform matrix applied in the data processing step
|
||||
applied_scale: Scale used in the data processing step
|
||||
camera_convention: Camera system convention used for the transformed poses
|
||||
Returns:
|
||||
Original poses
|
||||
"""
|
||||
output_poses = torch.cat(
|
||||
(
|
||||
poses,
|
||||
torch.tensor([[[0, 0, 0, 1]]], dtype=poses.dtype, device=poses.device).repeat_interleave(len(poses), 0),
|
||||
),
|
||||
1,
|
||||
)
|
||||
output_poses[..., :3, 3] /= applied_scale
|
||||
inv_transform = torch.linalg.inv(
|
||||
torch.cat(
|
||||
(
|
||||
applied_transform,
|
||||
torch.tensor([[0, 0, 0, 1]], dtype=applied_transform.dtype, device=applied_transform.device),
|
||||
),
|
||||
0,
|
||||
)
|
||||
)
|
||||
output_poses = torch.einsum("ij,bjk->bik", inv_transform, output_poses)
|
||||
if camera_convention == "opencv":
|
||||
output_poses[..., 0:3, 1:3] *= -1
|
||||
elif camera_convention == "opengl":
|
||||
pass
|
||||
else:
|
||||
raise ValueError(f"Camera convention {camera_convention} is not supported.")
|
||||
return output_poses[:, :3]
|
||||
@@ -78,18 +78,19 @@ def _parse_osm_txt(filename: str):
|
||||
return np.array([float(x) for x in nums]).reshape([4, 4]).astype(np.float32)
|
||||
|
||||
|
||||
def get_camera_params(scene_dir: str, split: str) -> Tuple[torch.Tensor, torch.Tensor, int]:
|
||||
def get_camera_params(
|
||||
scene_dir: str, split: Literal["train", "validation", "test"]
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, int]:
|
||||
"""Load camera intrinsic and extrinsic parameters for a given scene split.
|
||||
|
||||
Args"
|
||||
scene_dir : The directory containing the scene data.
|
||||
split : The split for which to load the camera parameters. Either "train", "val", or "test".
|
||||
split : The split for which to load the camera parameters.
|
||||
|
||||
Returns
|
||||
A tuple containing the intrinsic parameters (as a torch.Tensor of shape [N, 4, 4]),
|
||||
the camera-to-world matrices (as a torch.Tensor of shape [N, 4, 4]), and the number of cameras (N).
|
||||
"""
|
||||
split = "validation" if split == "val" else split
|
||||
split_dir = f"{scene_dir}/{split}"
|
||||
|
||||
# camera parameters files
|
||||
@@ -169,7 +170,7 @@ class NeRFOSR(DataParser):
|
||||
|
||||
# get all split cam params
|
||||
intrinsics_train, camera_to_worlds_train, n_train = get_camera_params(scene_dir, "train")
|
||||
intrinsics_val, camera_to_worlds_val, n_val = get_camera_params(scene_dir, "val")
|
||||
intrinsics_val, camera_to_worlds_val, n_val = get_camera_params(scene_dir, "validation")
|
||||
intrinsics_test, camera_to_worlds_test, _ = get_camera_params(scene_dir, "test")
|
||||
|
||||
# combine all cam params
|
||||
@@ -192,7 +193,7 @@ class NeRFOSR(DataParser):
|
||||
if split == "train":
|
||||
camera_to_worlds = camera_to_worlds[:n_train]
|
||||
intrinsics = intrinsics[:n_train]
|
||||
elif split == "val":
|
||||
elif split == "validation":
|
||||
camera_to_worlds = camera_to_worlds[n_train : n_train + n_val]
|
||||
intrinsics = intrinsics[n_train : n_train + n_val]
|
||||
elif split == "test":
|
||||
|
||||
@@ -138,6 +138,18 @@ class Trainer:
|
||||
'test': loads train/test datasets into memory
|
||||
'inference': does not load any dataset into memory
|
||||
"""
|
||||
# set up viewer if enabled
|
||||
viewer_log_path = self.base_dir / self.config.viewer.relative_log_filename
|
||||
self.viewer_state, banner_messages = None, None
|
||||
if self.config.is_viewer_enabled() and self.local_rank == 0:
|
||||
datapath = self.config.data
|
||||
if datapath is None:
|
||||
datapath = self.base_dir
|
||||
self.viewer_state, banner_messages = viewer_utils.setup_viewer(
|
||||
self.config.viewer, log_filename=viewer_log_path, datapath=datapath
|
||||
)
|
||||
self._check_viewer_warnings()
|
||||
|
||||
self.pipeline = self.config.pipeline.setup(
|
||||
device=self.device, test_mode=test_mode, world_size=self.world_size, local_rank=self.local_rank
|
||||
)
|
||||
@@ -153,17 +165,6 @@ class Trainer:
|
||||
)
|
||||
)
|
||||
|
||||
# set up viewer if enabled
|
||||
viewer_log_path = self.base_dir / self.config.viewer.relative_log_filename
|
||||
self.viewer_state, banner_messages = None, None
|
||||
if self.config.is_viewer_enabled() and self.local_rank == 0:
|
||||
datapath = self.pipeline.datamanager.get_datapath()
|
||||
if datapath is None:
|
||||
datapath = self.base_dir
|
||||
self.viewer_state, banner_messages = viewer_utils.setup_viewer(
|
||||
self.config.viewer, log_filename=viewer_log_path, datapath=datapath
|
||||
)
|
||||
self._check_viewer_warnings()
|
||||
# set up writers/profilers if enabled
|
||||
writer_log_path = self.base_dir / self.config.logging.relative_log_dir
|
||||
writer.setup_event_writer(
|
||||
|
||||
@@ -338,7 +338,7 @@ class MiDaSMSELoss(nn.Module):
|
||||
def __init__(self, reduction_type: Literal["image", "batch"] = "batch"):
|
||||
super().__init__()
|
||||
|
||||
self.reduction_type = reduction_type
|
||||
self.reduction_type: Literal["image", "batch"] = reduction_type
|
||||
# reduction here is different from the image/batch-based reduction. This is either "mean" or "sum"
|
||||
self.mse_loss = MSELoss(reduction="none")
|
||||
|
||||
|
||||
@@ -641,6 +641,7 @@ class NeuSSampler(Sampler):
|
||||
# Start with uniform sampling
|
||||
if ray_samples is None:
|
||||
ray_samples = self.uniform_sampler(ray_bundle, num_samples=self.num_samples)
|
||||
assert ray_samples is not None
|
||||
|
||||
total_iters = 0
|
||||
sorted_index = None
|
||||
@@ -697,6 +698,7 @@ class NeuSSampler(Sampler):
|
||||
"""
|
||||
batch_size = ray_samples.shape[0]
|
||||
prev_sdf, next_sdf = sdf[:, :-1], sdf[:, 1:]
|
||||
assert ray_samples.deltas is not None
|
||||
deltas = ray_samples.deltas[:, :-1, 0]
|
||||
mid_sdf = (prev_sdf + next_sdf) * 0.5
|
||||
cos_val = (next_sdf - prev_sdf) / (deltas + 1e-5)
|
||||
@@ -738,6 +740,9 @@ class NeuSSampler(Sampler):
|
||||
ray_samples_2 : ray_samples to merge
|
||||
"""
|
||||
|
||||
assert ray_samples_1.spacing_starts is not None and ray_samples_2.spacing_starts is not None
|
||||
assert ray_samples_1.spacing_ends is not None and ray_samples_2.spacing_ends is not None
|
||||
assert ray_samples_1.spacing_to_euclidean_fn is not None
|
||||
starts_1 = ray_samples_1.spacing_starts[..., 0]
|
||||
starts_2 = ray_samples_2.spacing_starts[..., 0]
|
||||
|
||||
|
||||
@@ -101,6 +101,7 @@ class Model(nn.Module):
|
||||
# NOTE: call `super().populate_modules()` in subclasses
|
||||
|
||||
if self.config.enable_collider:
|
||||
assert self.config.collider_params is not None
|
||||
self.collider = NearFarCollider(
|
||||
near_plane=self.config.collider_params["near_plane"], far_plane=self.config.collider_params["far_plane"]
|
||||
)
|
||||
|
||||
@@ -90,6 +90,8 @@ class TensoRFModel(Model):
|
||||
config: TensoRF configuration to instantiate model
|
||||
"""
|
||||
|
||||
config: TensoRFModelConfig
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: TensoRFModelConfig,
|
||||
@@ -303,6 +305,7 @@ class TensoRFModel(Model):
|
||||
image = batch["image"].to(outputs["rgb"].device)
|
||||
rgb = outputs["rgb"]
|
||||
acc = colormaps.apply_colormap(outputs["accumulation"])
|
||||
assert self.config.collider_params is not None
|
||||
depth = colormaps.apply_depth_colormap(
|
||||
outputs["depth"],
|
||||
accumulation=outputs["accumulation"],
|
||||
|
||||
@@ -201,6 +201,7 @@ class NeRFModel(Model):
|
||||
rgb_fine = outputs["rgb_fine"]
|
||||
acc_coarse = colormaps.apply_colormap(outputs["accumulation_coarse"])
|
||||
acc_fine = colormaps.apply_colormap(outputs["accumulation_fine"])
|
||||
assert self.config.collider_params is not None
|
||||
depth_coarse = colormaps.apply_depth_colormap(
|
||||
outputs["depth_coarse"],
|
||||
accumulation=outputs["accumulation_coarse"],
|
||||
|
||||
@@ -95,6 +95,7 @@ class Pipeline(nn.Module):
|
||||
|
||||
datamanager: DataManager
|
||||
_model: Model
|
||||
world_size: int
|
||||
|
||||
@property
|
||||
def model(self):
|
||||
@@ -201,6 +202,7 @@ class VanillaPipelineConfig(cfg.InstantiateConfig):
|
||||
class VanillaPipeline(Pipeline):
|
||||
"""The pipeline class for the vanilla nerf setup of multiple cameras for one or a few scenes.
|
||||
|
||||
Args:
|
||||
config: configuration to instantiate pipeline
|
||||
device: location to place model and data
|
||||
test_mode:
|
||||
|
||||
@@ -75,6 +75,11 @@ def run_hloc(
|
||||
camera_model: Camera model to use.
|
||||
gpu: If True, use GPU.
|
||||
verbose: If True, logs the output of the command.
|
||||
matching_method: Method to use for matching images.
|
||||
feature_type: Type of visual features to use.
|
||||
matcher_type: Type of feature matcher to use.
|
||||
num_matched: Number of image pairs for loc.
|
||||
refine_pixsfm: If True, refine the reconstruction using pixel-perfect-sfm.
|
||||
"""
|
||||
if not _HAS_HLOC:
|
||||
CONSOLE.print(
|
||||
|
||||
@@ -27,7 +27,7 @@ from nerfstudio.process_data.process_data_utils import CAMERA_MODELS
|
||||
CONSOLE = Console(width=120)
|
||||
|
||||
|
||||
def _find_distortion_param(calib_xml: ET.Element, param_name: str):
|
||||
def _find_param(calib_xml: ET.Element, param_name: str):
|
||||
param = calib_xml.find(param_name)
|
||||
if param is not None:
|
||||
return float(param.text) # type: ignore
|
||||
@@ -62,34 +62,71 @@ def metashape_to_json( # pylint: disable=too-many-statements
|
||||
raise ValueError("No sensors found")
|
||||
|
||||
calibrated_sensors = [sensor for sensor in sensors if sensor.find("calibration")]
|
||||
if len(calibrated_sensors) != 1:
|
||||
raise ValueError("Only one sensor is supported for now")
|
||||
|
||||
sensor = calibrated_sensors[0]
|
||||
|
||||
if not calibrated_sensors:
|
||||
raise ValueError("No calibrated sensor found in Metashape XML")
|
||||
sensor_type = [s.get("type") for s in calibrated_sensors]
|
||||
if sensor_type.count(sensor_type[0]) != len(sensor_type):
|
||||
raise ValueError("All Metashape sensors do not have the same sensor type")
|
||||
data = {}
|
||||
if sensor_type[0] == "frame":
|
||||
data["camera_model"] = CAMERA_MODELS["perspective"].value
|
||||
elif sensor_type[0] == "fisheye":
|
||||
data["camera_model"] = CAMERA_MODELS["fisheye"].value
|
||||
elif sensor_type[0] == "spherical":
|
||||
data["camera_model"] = CAMERA_MODELS["equirectangular"].value
|
||||
else:
|
||||
# Cylindrical and RPC sensor types are not supported
|
||||
raise ValueError(f"Unsupported Metashape sensor type '{sensor_type[0]}'")
|
||||
|
||||
assert sensor is not None, "Sensor not found in Metashape XML"
|
||||
resolution = sensor.find("resolution")
|
||||
assert resolution is not None, "Resolution not found in Metashape xml"
|
||||
data["w"] = int(resolution.get("width")) # type: ignore
|
||||
data["h"] = int(resolution.get("height")) # type: ignore
|
||||
sensor_dict = {}
|
||||
for sensor in calibrated_sensors:
|
||||
s = {}
|
||||
resolution = sensor.find("resolution")
|
||||
assert resolution is not None, "Resolution not found in Metashape xml"
|
||||
s["w"] = int(resolution.get("width")) # type: ignore
|
||||
s["h"] = int(resolution.get("height")) # type: ignore
|
||||
|
||||
calib = sensor.find("calibration")
|
||||
assert calib is not None, "Calibration not found in Metashape xml"
|
||||
data["fl_x"] = float(calib.find("f").text) # type: ignore
|
||||
data["fl_y"] = float(calib.find("f").text) # type: ignore
|
||||
data["cx"] = float(calib.find("cx").text) + data["w"] / 2.0 # type: ignore
|
||||
data["cy"] = float(calib.find("cy").text) + data["h"] / 2.0 # type: ignore
|
||||
calib = sensor.find("calibration")
|
||||
f = calib.find("f")
|
||||
if f is not None:
|
||||
s["fl_x"] = s["fl_y"] = float(f.text) # type: ignore
|
||||
s["cx"] = _find_param(calib, "cx") + s["w"] / 2.0 # type: ignore
|
||||
s["cy"] = _find_param(calib, "cy") + s["h"] / 2.0 # type: ignore
|
||||
|
||||
data["k1"] = _find_distortion_param(calib, "k1")
|
||||
data["k2"] = _find_distortion_param(calib, "k2")
|
||||
data["k3"] = _find_distortion_param(calib, "k3")
|
||||
data["k4"] = _find_distortion_param(calib, "k4")
|
||||
data["p1"] = _find_distortion_param(calib, "p1")
|
||||
data["p2"] = _find_distortion_param(calib, "p2")
|
||||
s["k1"] = _find_param(calib, "k1")
|
||||
s["k2"] = _find_param(calib, "k2")
|
||||
s["k3"] = _find_param(calib, "k3")
|
||||
s["k4"] = _find_param(calib, "k4")
|
||||
s["p1"] = _find_param(calib, "p1")
|
||||
s["p2"] = _find_param(calib, "p2")
|
||||
|
||||
data["camera_model"] = CAMERA_MODELS["perspective"].value
|
||||
sensor_dict[sensor.get("id")] = s
|
||||
|
||||
components = chunk.find("components")
|
||||
component_dict = {}
|
||||
for component in components:
|
||||
transform = component.find("transform")
|
||||
if transform is not None:
|
||||
rotation = transform.find("rotation")
|
||||
if rotation is None:
|
||||
r = np.eye(3)
|
||||
else:
|
||||
r = np.array([float(x) for x in rotation.text.split()]).reshape((3, 3))
|
||||
translation = transform.find("translation")
|
||||
if translation is None:
|
||||
t = np.zeros(3)
|
||||
else:
|
||||
t = np.array([float(x) for x in translation.text.split()])
|
||||
scale = transform.find("scale")
|
||||
if scale is None:
|
||||
s = 1.0
|
||||
else:
|
||||
s = float(scale.text)
|
||||
|
||||
m = np.eye(4)
|
||||
m[:3, :3] = r
|
||||
m[:3, 3] = t / s
|
||||
component_dict[component.get("id")] = m
|
||||
|
||||
frames = []
|
||||
cameras = chunk.find("cameras")
|
||||
@@ -97,33 +134,36 @@ def metashape_to_json( # pylint: disable=too-many-statements
|
||||
num_skipped = 0
|
||||
for camera in cameras:
|
||||
frame = {}
|
||||
# Labels sometimes have a file extension. We remove it for consistency.
|
||||
camera_label = camera.get("label").split(".")[0] # type: ignore
|
||||
camera_label = camera.get("label")
|
||||
if camera_label not in image_filename_map:
|
||||
continue
|
||||
# Labels sometimes have a file extension. Try without the extension.
|
||||
# (maybe it's just a '.' in the image name)
|
||||
camera_label = camera_label.split(".")[0] # type: ignore
|
||||
if camera_label not in image_filename_map:
|
||||
continue
|
||||
frame["file_path"] = image_filename_map[camera_label].as_posix()
|
||||
|
||||
if camera.get("sensor_id") != sensor.get("id"):
|
||||
sensor_id = camera.get("sensor_id")
|
||||
if sensor_id not in sensor_dict:
|
||||
# this should only happen when we have a sensor that doesn't have calibration
|
||||
if verbose:
|
||||
CONSOLE.print(f"Missing sensor calibration for {camera.get('label')}, Skipping")
|
||||
num_skipped += 1
|
||||
continue
|
||||
# Add all sensor parameters to this frame.
|
||||
frame.update(sensor_dict[sensor_id])
|
||||
|
||||
if camera.find("transform") is None:
|
||||
if verbose:
|
||||
CONSOLE.print(f"Missing transforms data for {camera.get('label')}, Skipping")
|
||||
num_skipped += 1
|
||||
continue
|
||||
t = [float(x) for x in camera.find("transform").text.split()] # type: ignore
|
||||
transform = np.array(
|
||||
[
|
||||
[t[8], -t[9], -t[10], t[11]],
|
||||
[t[0], -t[1], -t[2], t[3]],
|
||||
[t[4], -t[5], -t[6], t[7]],
|
||||
[t[12], -t[13], -t[14], t[15]],
|
||||
]
|
||||
)
|
||||
transform = np.array([float(x) for x in camera.find("transform").text.split()]).reshape((4, 4))
|
||||
component_id = camera.get("component_id")
|
||||
if component_id in component_dict:
|
||||
transform = component_dict[component_id] @ transform
|
||||
transform = transform[[2, 0, 1, 3], :]
|
||||
transform[:, 1:3] *= -1
|
||||
frame["transform_matrix"] = transform.tolist()
|
||||
frames.append(frame)
|
||||
|
||||
|
||||
@@ -159,7 +159,9 @@ def convert_video_to_images(
|
||||
ffmpeg_cmd += f" -vf thumbnail={spacing},setpts=N/TB{crop_cmd} -r 1"
|
||||
else:
|
||||
CONSOLE.print("[bold red]Can't satisfy requested number of frames. Extracting all frames.")
|
||||
ffmpeg_cmd += f" -pix_fmt bgr8 -vf {crop_cmd[1:]}"
|
||||
ffmpeg_cmd += " -pix_fmt bgr8"
|
||||
if crop_cmd != "":
|
||||
ffmpeg_cmd += f" -vf {crop_cmd[1:]}"
|
||||
|
||||
ffmpeg_cmd += f" {out_filename}"
|
||||
run_command(ffmpeg_cmd, verbose=verbose)
|
||||
|
||||
@@ -60,7 +60,7 @@ def realitycapture_to_json(
|
||||
cameras.setdefault(column, []).append(value)
|
||||
|
||||
for name in cameras["#name"]:
|
||||
camera_label = name.split(".")[0]
|
||||
camera_label = name.rpartition(".")[0]
|
||||
if camera_label in image_filename_map:
|
||||
img = np.array(Image.open(output_dir / image_filename_map[camera_label]))
|
||||
break
|
||||
@@ -73,7 +73,7 @@ def realitycapture_to_json(
|
||||
missing_image_data = 0
|
||||
|
||||
for i, name in enumerate(cameras["#name"]):
|
||||
basename = name.split(".")[0]
|
||||
basename = name.rpartition(".")[0]
|
||||
if basename not in image_filename_map:
|
||||
if verbose:
|
||||
CONSOLE.print(f"Missing image for camera data {basename}, Skipping")
|
||||
|
||||
@@ -670,8 +670,17 @@ class ViewerState:
|
||||
|
||||
image = selected_output[..., [2, 1, 0]].cpu().numpy()
|
||||
|
||||
data = cv2.imencode(".jpg", image, [cv2.IMWRITE_JPEG_QUALITY, 75])[1].tobytes()
|
||||
data = str("data:image/jpeg;base64," + base64.b64encode(data).decode("ascii"))
|
||||
data = cv2.imencode(
|
||||
f".{self.config.image_format}",
|
||||
image,
|
||||
[
|
||||
cv2.IMWRITE_JPEG_QUALITY,
|
||||
self.config.jpeg_quality,
|
||||
cv2.IMWRITE_PNG_COMPRESSION,
|
||||
self.config.png_compression,
|
||||
],
|
||||
)[1].tobytes()
|
||||
data = str(f"data:image/{self.config.image_format};base64," + base64.b64encode(data).decode("ascii"))
|
||||
self.vis["render_img"].write(data)
|
||||
|
||||
def _update_viewer_stats(self, render_time: float, num_rays: int, image_height: int, image_width: int) -> None:
|
||||
|
||||
@@ -12,12 +12,30 @@ from nerfstudio.data.dataparsers.nerfstudio_dataparser import NerfstudioDataPars
|
||||
from nerfstudio.data.utils.colmap_parsing_utils import Camera
|
||||
from nerfstudio.data.utils.colmap_parsing_utils import Image as ColmapImage
|
||||
from nerfstudio.data.utils.colmap_parsing_utils import (
|
||||
qvec2rotmat,
|
||||
write_cameras_binary,
|
||||
write_images_binary,
|
||||
)
|
||||
from scripts.process_data import ProcessImages
|
||||
|
||||
|
||||
def random_quaternion(num_poses: int):
|
||||
"""
|
||||
Generates random rotation quatenion.
|
||||
"""
|
||||
u, v, w = np.random.uniform(size=(3, num_poses))
|
||||
quaternion = np.stack(
|
||||
(
|
||||
np.sqrt(1 - u) * np.sin(2 * np.pi * v),
|
||||
np.sqrt(1 - u) * np.cos(2 * np.pi * v),
|
||||
np.sqrt(u) * np.sin(2 * np.pi * w),
|
||||
np.sqrt(u) * np.cos(2 * np.pi * w),
|
||||
),
|
||||
-1,
|
||||
)
|
||||
return quaternion
|
||||
|
||||
|
||||
def test_process_images_skip_colmap(tmp_path: Path):
|
||||
"""
|
||||
Test ns-process-data images
|
||||
@@ -34,21 +52,21 @@ def test_process_images_skip_colmap(tmp_path: Path):
|
||||
)
|
||||
frames = {}
|
||||
num_frames = 10
|
||||
qvecs = np.random.uniform(size=(num_frames, 4))
|
||||
qvecs = random_quaternion(num_frames)
|
||||
tvecs = np.random.uniform(size=(num_frames, 3))
|
||||
# original_poses = np.concatenate(
|
||||
# (
|
||||
# np.concatenate(
|
||||
# (
|
||||
# np.stack(list(map(qvec2rotmat, qvecs))),
|
||||
# tvecs[:, :, None],
|
||||
# ),
|
||||
# -1,
|
||||
# ),
|
||||
# np.array([[[0, 0, 0, 1]]], dtype=qvecs.dtype).repeat(num_frames, 0),
|
||||
# ),
|
||||
# -2,
|
||||
# )
|
||||
original_poses = np.concatenate(
|
||||
(
|
||||
np.concatenate(
|
||||
(
|
||||
np.stack(list(map(qvec2rotmat, qvecs))),
|
||||
tvecs[:, :, None],
|
||||
),
|
||||
-1,
|
||||
),
|
||||
np.array([[[0, 0, 0, 1]]], dtype=qvecs.dtype).repeat(num_frames, 0),
|
||||
),
|
||||
-2,
|
||||
)
|
||||
for i in range(num_frames):
|
||||
frames[i + 1] = ColmapImage(i + 1, qvecs[i], tvecs[i], 1, f"image_{i}.png", [], [])
|
||||
Image.new("RGB", (width, height)).save(tmp_path / "images" / f"image_{i}.png")
|
||||
@@ -77,4 +95,12 @@ def test_process_images_skip_colmap(tmp_path: Path):
|
||||
outputs = parser.get_dataparser_outputs("train")
|
||||
assert len(outputs.image_filenames) == 9
|
||||
assert torch.is_tensor(outputs.dataparser_transform)
|
||||
# TODO @jkulhanek: Add tests if the loaded poses are the same
|
||||
|
||||
# Test if the original poses can be obtained back
|
||||
gt_poses = original_poses[[0, 1, 2, 3, 4, 5, 6, 7, 9]]
|
||||
dataparser_poses = outputs.transform_poses_to_original_space(outputs.cameras.camera_to_worlds, "opencv").numpy()
|
||||
dataparser_poses = np.concatenate(
|
||||
(dataparser_poses, np.array([[[0, 0, 0, 1]]]).repeat(len(dataparser_poses), 0)), 1
|
||||
)
|
||||
dataparser_poses = np.linalg.inv(dataparser_poses)
|
||||
np.testing.assert_allclose(gt_poses, dataparser_poses, rtol=0, atol=1e-5)
|
||||
Reference in new issue
Block a user