From c0464f8a9113636f877841302a0e48613cb662f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=C3=A1=C5=A1=20Kulh=C3=A1nek?= Date: Sun, 19 Mar 2023 12:09:33 +0100 Subject: [PATCH 1/8] Adds util function to convert transformed poses back + tests (#1594) Improves tests of ns-process-data images + Adds util function to convert poses back from transformed coordinates. --- .../data/dataparsers/base_dataparser.py | 64 +++++++++++++++++++ tests/process_data/test_process_images.py | 56 +++++++++++----- 2 files changed, 105 insertions(+), 15 deletions(-) diff --git a/nerfstudio/data/dataparsers/base_dataparser.py b/nerfstudio/data/dataparsers/base_dataparser.py index 89c7c4e21..12367e51f 100644 --- a/nerfstudio/data/dataparsers/base_dataparser.py +++ b/nerfstudio/data/dataparsers/base_dataparser.py @@ -24,6 +24,7 @@ from typing import Any, Dict, List, Optional, Type import torch from torchtyping import TensorType +from typing_extensions import Literal import nerfstudio.configs.base_config as cfg from nerfstudio.cameras.cameras import Cameras @@ -89,6 +90,26 @@ class DataparserOutputs: with open(path, "w", encoding="UTF-8") as file: json.dump(data, file, indent=4) + def transform_poses_to_original_space( + self, + poses: TensorType["num_poses", 3, 4], + camera_convention: Literal["opengl", "opencv"] = "opencv", + ) -> TensorType["num_poses", 3, 4]: + """ + Transforms the poses in the transformed space back to the original world coordinate system. + Args: + poses: Poses in the transformed space + camera_convention: Camera system convention used for the transformed poses + Returns: + Original poses + """ + return transform_poses_to_original_space( + poses, + self.dataparser_transform, + self.dataparser_scale, + camera_convention=camera_convention, + ) + @dataclass class DataParserConfig(cfg.InstantiateConfig): @@ -136,3 +157,46 @@ class DataParser: """ dataparser_outputs = self._generate_dataparser_outputs(split) return dataparser_outputs + + +def transform_poses_to_original_space( + poses: TensorType["num_poses", 3, 4], + applied_transform: TensorType[3, 4], + applied_scale: float, + camera_convention: Literal["opengl", "opencv"] = "opencv", +) -> TensorType["num_poses", 3, 4]: + """ + Transforms the poses in the transformed space back to the original world coordinate system. + Args: + poses: Poses in the transformed space + applied_transform: Transform matrix applied in the data processing step + applied_scale: Scale used in the data processing step + camera_convention: Camera system convention used for the transformed poses + Returns: + Original poses + """ + output_poses = torch.cat( + ( + poses, + torch.tensor([[[0, 0, 0, 1]]], dtype=poses.dtype, device=poses.device).repeat_interleave(len(poses), 0), + ), + 1, + ) + output_poses[..., :3, 3] /= applied_scale + inv_transform = torch.linalg.inv( + torch.cat( + ( + applied_transform, + torch.tensor([[0, 0, 0, 1]], dtype=applied_transform.dtype, device=applied_transform.device), + ), + 0, + ) + ) + output_poses = torch.einsum("ij,bjk->bik", inv_transform, output_poses) + if camera_convention == "opencv": + output_poses[..., 0:3, 1:3] *= -1 + elif camera_convention == "opengl": + pass + else: + raise ValueError(f"Camera convention {camera_convention} is not supported.") + return output_poses[:, :3] diff --git a/tests/process_data/test_process_images.py b/tests/process_data/test_process_images.py index 3e4fa82a9..1b86a5adb 100644 --- a/tests/process_data/test_process_images.py +++ b/tests/process_data/test_process_images.py @@ -12,12 +12,30 @@ from nerfstudio.data.dataparsers.nerfstudio_dataparser import NerfstudioDataPars from nerfstudio.data.utils.colmap_parsing_utils import Camera from nerfstudio.data.utils.colmap_parsing_utils import Image as ColmapImage from nerfstudio.data.utils.colmap_parsing_utils import ( + qvec2rotmat, write_cameras_binary, write_images_binary, ) from scripts.process_data import ProcessImages +def random_quaternion(num_poses: int): + """ + Generates random rotation quatenion. + """ + u, v, w = np.random.uniform(size=(3, num_poses)) + quaternion = np.stack( + ( + np.sqrt(1 - u) * np.sin(2 * np.pi * v), + np.sqrt(1 - u) * np.cos(2 * np.pi * v), + np.sqrt(u) * np.sin(2 * np.pi * w), + np.sqrt(u) * np.cos(2 * np.pi * w), + ), + -1, + ) + return quaternion + + def test_process_images_skip_colmap(tmp_path: Path): """ Test ns-process-data images @@ -34,21 +52,21 @@ def test_process_images_skip_colmap(tmp_path: Path): ) frames = {} num_frames = 10 - qvecs = np.random.uniform(size=(num_frames, 4)) + qvecs = random_quaternion(num_frames) tvecs = np.random.uniform(size=(num_frames, 3)) - # original_poses = np.concatenate( - # ( - # np.concatenate( - # ( - # np.stack(list(map(qvec2rotmat, qvecs))), - # tvecs[:, :, None], - # ), - # -1, - # ), - # np.array([[[0, 0, 0, 1]]], dtype=qvecs.dtype).repeat(num_frames, 0), - # ), - # -2, - # ) + original_poses = np.concatenate( + ( + np.concatenate( + ( + np.stack(list(map(qvec2rotmat, qvecs))), + tvecs[:, :, None], + ), + -1, + ), + np.array([[[0, 0, 0, 1]]], dtype=qvecs.dtype).repeat(num_frames, 0), + ), + -2, + ) for i in range(num_frames): frames[i + 1] = ColmapImage(i + 1, qvecs[i], tvecs[i], 1, f"image_{i}.png", [], []) Image.new("RGB", (width, height)).save(tmp_path / "images" / f"image_{i}.png") @@ -77,4 +95,12 @@ def test_process_images_skip_colmap(tmp_path: Path): outputs = parser.get_dataparser_outputs("train") assert len(outputs.image_filenames) == 9 assert torch.is_tensor(outputs.dataparser_transform) - # TODO @jkulhanek: Add tests if the loaded poses are the same + + # Test if the original poses can be obtained back + gt_poses = original_poses[[0, 1, 2, 3, 4, 5, 6, 7, 9]] + dataparser_poses = outputs.transform_poses_to_original_space(outputs.cameras.camera_to_worlds, "opencv").numpy() + dataparser_poses = np.concatenate( + (dataparser_poses, np.array([[[0, 0, 0, 1]]]).repeat(len(dataparser_poses), 0)), 1 + ) + dataparser_poses = np.linalg.inv(dataparser_poses) + np.testing.assert_allclose(gt_poses, dataparser_poses, rtol=0, atol=1e-5) From 81e39cb276e4600100ef3115b2c41dee163d60f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Devernay?= <124616582+f-dy@users.noreply.github.com> Date: Sun, 19 Mar 2023 10:21:57 -0700 Subject: [PATCH 2/8] improve metashape data import (#1605) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * improve metashape data import - support multiple sensors - support fisheye and equirectangular sensors - support multiple components with transforms - fix bug when image name has a '.' in it, excluding the prefix separator * metashape_util: fix string formatting --------- Co-authored-by: Frédéric Devernay Co-authored-by: Matthew Tancik --- nerfstudio/process_data/metashape_utils.py | 114 ++++++++++++++------- 1 file changed, 77 insertions(+), 37 deletions(-) diff --git a/nerfstudio/process_data/metashape_utils.py b/nerfstudio/process_data/metashape_utils.py index 17d7771e8..0e4a7593e 100644 --- a/nerfstudio/process_data/metashape_utils.py +++ b/nerfstudio/process_data/metashape_utils.py @@ -27,7 +27,7 @@ from nerfstudio.process_data.process_data_utils import CAMERA_MODELS CONSOLE = Console(width=120) -def _find_distortion_param(calib_xml: ET.Element, param_name: str): +def _find_param(calib_xml: ET.Element, param_name: str): param = calib_xml.find(param_name) if param is not None: return float(param.text) # type: ignore @@ -62,34 +62,71 @@ def metashape_to_json( # pylint: disable=too-many-statements raise ValueError("No sensors found") calibrated_sensors = [sensor for sensor in sensors if sensor.find("calibration")] - if len(calibrated_sensors) != 1: - raise ValueError("Only one sensor is supported for now") - - sensor = calibrated_sensors[0] - + if not calibrated_sensors: + raise ValueError("No calibrated sensor found in Metashape XML") + sensor_type = [s.get("type") for s in calibrated_sensors] + if sensor_type.count(sensor_type[0]) != len(sensor_type): + raise ValueError("All Metashape sensors do not have the same sensor type") data = {} + if sensor_type[0] == "frame": + data["camera_model"] = CAMERA_MODELS["perspective"].value + elif sensor_type[0] == "fisheye": + data["camera_model"] = CAMERA_MODELS["fisheye"].value + elif sensor_type[0] == "spherical": + data["camera_model"] = CAMERA_MODELS["equirectangular"].value + else: + # Cylindrical and RPC sensor types are not supported + raise ValueError(f"Unsupported Metashape sensor type '{sensor_type[0]}'") - assert sensor is not None, "Sensor not found in Metashape XML" - resolution = sensor.find("resolution") - assert resolution is not None, "Resolution not found in Metashape xml" - data["w"] = int(resolution.get("width")) # type: ignore - data["h"] = int(resolution.get("height")) # type: ignore + sensor_dict = {} + for sensor in calibrated_sensors: + s = {} + resolution = sensor.find("resolution") + assert resolution is not None, "Resolution not found in Metashape xml" + s["w"] = int(resolution.get("width")) # type: ignore + s["h"] = int(resolution.get("height")) # type: ignore - calib = sensor.find("calibration") - assert calib is not None, "Calibration not found in Metashape xml" - data["fl_x"] = float(calib.find("f").text) # type: ignore - data["fl_y"] = float(calib.find("f").text) # type: ignore - data["cx"] = float(calib.find("cx").text) + data["w"] / 2.0 # type: ignore - data["cy"] = float(calib.find("cy").text) + data["h"] / 2.0 # type: ignore + calib = sensor.find("calibration") + f = calib.find("f") + if f is not None: + s["fl_x"] = s["fl_y"] = float(f.text) # type: ignore + s["cx"] = _find_param(calib, "cx") + s["w"] / 2.0 # type: ignore + s["cy"] = _find_param(calib, "cy") + s["h"] / 2.0 # type: ignore - data["k1"] = _find_distortion_param(calib, "k1") - data["k2"] = _find_distortion_param(calib, "k2") - data["k3"] = _find_distortion_param(calib, "k3") - data["k4"] = _find_distortion_param(calib, "k4") - data["p1"] = _find_distortion_param(calib, "p1") - data["p2"] = _find_distortion_param(calib, "p2") + s["k1"] = _find_param(calib, "k1") + s["k2"] = _find_param(calib, "k2") + s["k3"] = _find_param(calib, "k3") + s["k4"] = _find_param(calib, "k4") + s["p1"] = _find_param(calib, "p1") + s["p2"] = _find_param(calib, "p2") - data["camera_model"] = CAMERA_MODELS["perspective"].value + sensor_dict[sensor.get("id")] = s + + components = chunk.find("components") + component_dict = {} + for component in components: + transform = component.find("transform") + if transform is not None: + rotation = transform.find("rotation") + if rotation is None: + r = np.eye(3) + else: + r = np.array([float(x) for x in rotation.text.split()]).reshape((3, 3)) + translation = transform.find("translation") + if translation is None: + t = np.zeros(3) + else: + t = np.array([float(x) for x in translation.text.split()]) + scale = transform.find("scale") + if scale is None: + s = 1.0 + else: + s = float(scale.text) + + m = np.eye(4) + m[:3, :3] = r + m[:3, 3] = t / s + component_dict[component.get("id")] = m frames = [] cameras = chunk.find("cameras") @@ -97,33 +134,36 @@ def metashape_to_json( # pylint: disable=too-many-statements num_skipped = 0 for camera in cameras: frame = {} - # Labels sometimes have a file extension. We remove it for consistency. - camera_label = camera.get("label").split(".")[0] # type: ignore + camera_label = camera.get("label") if camera_label not in image_filename_map: - continue + # Labels sometimes have a file extension. Try without the extension. + # (maybe it's just a '.' in the image name) + camera_label = camera_label.split(".")[0] # type: ignore + if camera_label not in image_filename_map: + continue frame["file_path"] = image_filename_map[camera_label].as_posix() - if camera.get("sensor_id") != sensor.get("id"): + sensor_id = camera.get("sensor_id") + if sensor_id not in sensor_dict: # this should only happen when we have a sensor that doesn't have calibration if verbose: CONSOLE.print(f"Missing sensor calibration for {camera.get('label')}, Skipping") num_skipped += 1 continue + # Add all sensor parameters to this frame. + frame.update(sensor_dict[sensor_id]) if camera.find("transform") is None: if verbose: CONSOLE.print(f"Missing transforms data for {camera.get('label')}, Skipping") num_skipped += 1 continue - t = [float(x) for x in camera.find("transform").text.split()] # type: ignore - transform = np.array( - [ - [t[8], -t[9], -t[10], t[11]], - [t[0], -t[1], -t[2], t[3]], - [t[4], -t[5], -t[6], t[7]], - [t[12], -t[13], -t[14], t[15]], - ] - ) + transform = np.array([float(x) for x in camera.find("transform").text.split()]).reshape((4, 4)) + component_id = camera.get("component_id") + if component_id in component_dict: + transform = component_dict[component_id] @ transform + transform = transform[[2, 0, 1, 3], :] + transform[:, 1:3] *= -1 frame["transform_matrix"] = transform.tolist() frames.append(frame) From c65760121fbb0a19dff52e6ac577de6b2019f70e Mon Sep 17 00:00:00 2001 From: Jackson Waschura <35237687+jackson-waschura@users.noreply.github.com> Date: Tue, 21 Mar 2023 11:48:43 -0700 Subject: [PATCH 3/8] Fixed incorrect ffmpeg command construction (#1645) Co-authored-by: jackson-waschura --- nerfstudio/process_data/process_data_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nerfstudio/process_data/process_data_utils.py b/nerfstudio/process_data/process_data_utils.py index a73127fcb..817e2c6cd 100644 --- a/nerfstudio/process_data/process_data_utils.py +++ b/nerfstudio/process_data/process_data_utils.py @@ -159,7 +159,9 @@ def convert_video_to_images( ffmpeg_cmd += f" -vf thumbnail={spacing},setpts=N/TB{crop_cmd} -r 1" else: CONSOLE.print("[bold red]Can't satisfy requested number of frames. Extracting all frames.") - ffmpeg_cmd += f" -pix_fmt bgr8 -vf {crop_cmd[1:]}" + ffmpeg_cmd += " -pix_fmt bgr8" + if crop_cmd != "": + ffmpeg_cmd += f" -vf {crop_cmd[1:]}" ffmpeg_cmd += f" {out_filename}" run_command(ffmpeg_cmd, verbose=verbose) From e7c3edce0e0af061012c707050dc451b79ca4211 Mon Sep 17 00:00:00 2001 From: James Gardner Date: Tue, 21 Mar 2023 22:41:59 +0000 Subject: [PATCH 4/8] Typo in nerfosr validation set (#1631) * fix nerfosr validation set * fix nerfosr validation set * fix nerfosr validation set * fix nerfosr function type hints --- nerfstudio/data/dataparsers/nerfosr_dataparser.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/nerfstudio/data/dataparsers/nerfosr_dataparser.py b/nerfstudio/data/dataparsers/nerfosr_dataparser.py index 6b24ca957..820a28956 100644 --- a/nerfstudio/data/dataparsers/nerfosr_dataparser.py +++ b/nerfstudio/data/dataparsers/nerfosr_dataparser.py @@ -78,18 +78,19 @@ def _parse_osm_txt(filename: str): return np.array([float(x) for x in nums]).reshape([4, 4]).astype(np.float32) -def get_camera_params(scene_dir: str, split: str) -> Tuple[torch.Tensor, torch.Tensor, int]: +def get_camera_params( + scene_dir: str, split: Literal["train", "validation", "test"] +) -> Tuple[torch.Tensor, torch.Tensor, int]: """Load camera intrinsic and extrinsic parameters for a given scene split. Args" scene_dir : The directory containing the scene data. - split : The split for which to load the camera parameters. Either "train", "val", or "test". + split : The split for which to load the camera parameters. Returns A tuple containing the intrinsic parameters (as a torch.Tensor of shape [N, 4, 4]), the camera-to-world matrices (as a torch.Tensor of shape [N, 4, 4]), and the number of cameras (N). """ - split = "validation" if split == "val" else split split_dir = f"{scene_dir}/{split}" # camera parameters files @@ -169,7 +170,7 @@ class NeRFOSR(DataParser): # get all split cam params intrinsics_train, camera_to_worlds_train, n_train = get_camera_params(scene_dir, "train") - intrinsics_val, camera_to_worlds_val, n_val = get_camera_params(scene_dir, "val") + intrinsics_val, camera_to_worlds_val, n_val = get_camera_params(scene_dir, "validation") intrinsics_test, camera_to_worlds_test, _ = get_camera_params(scene_dir, "test") # combine all cam params @@ -192,7 +193,7 @@ class NeRFOSR(DataParser): if split == "train": camera_to_worlds = camera_to_worlds[:n_train] intrinsics = intrinsics[:n_train] - elif split == "val": + elif split == "validation": camera_to_worlds = camera_to_worlds[n_train : n_train + n_val] intrinsics = intrinsics[n_train : n_train + n_val] elif split == "test": From 6db1d314cc646598daf0093b3a0513bbb43b718f Mon Sep 17 00:00:00 2001 From: machenmusik Date: Tue, 21 Mar 2023 19:04:12 -0400 Subject: [PATCH 5/8] add quality setting; allow png for lossless compression (#1639) * add quality setting; allow png for lossless compression * black reformatting * JPEG default, 90 quality --- nerfstudio/configs/base_config.py | 9 +++++++++ nerfstudio/viewer/server/viewer_utils.py | 13 +++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/nerfstudio/configs/base_config.py b/nerfstudio/configs/base_config.py index fda6cc10f..3d7872d74 100644 --- a/nerfstudio/configs/base_config.py +++ b/nerfstudio/configs/base_config.py @@ -23,11 +23,14 @@ from dataclasses import dataclass from pathlib import Path from typing import Any, List, Optional, Tuple, Type +from typing_extensions import Literal + # model instances from nerfstudio.utils import writer warnings.filterwarnings("ignore", module="torchvision") + # Pretty printing class class PrintableConfig: # pylint: disable=too-few-public-methods """Printable Config defining str function""" @@ -148,3 +151,9 @@ class ViewerConfig(PrintableConfig): actually used in training/evaluation. If -1, display all.""" quit_on_train_completion: bool = False """Whether to kill the training job when it has completed. Note this will stop rendering in the viewer.""" + image_format: Literal["jpeg", "png"] = "jpeg" + """Image format viewer should use; jpeg is lossy compression, while png is lossless.""" + jpeg_quality: int = 90 + """Quality tradeoff to use for jpeg compression.""" + png_compression: int = 1 + """Size/speed tradeoff to use for png compression.""" diff --git a/nerfstudio/viewer/server/viewer_utils.py b/nerfstudio/viewer/server/viewer_utils.py index b781bea08..a6ff0a412 100644 --- a/nerfstudio/viewer/server/viewer_utils.py +++ b/nerfstudio/viewer/server/viewer_utils.py @@ -670,8 +670,17 @@ class ViewerState: image = selected_output[..., [2, 1, 0]].cpu().numpy() - data = cv2.imencode(".jpg", image, [cv2.IMWRITE_JPEG_QUALITY, 75])[1].tobytes() - data = str("data:image/jpeg;base64," + base64.b64encode(data).decode("ascii")) + data = cv2.imencode( + f".{self.config.image_format}", + image, + [ + cv2.IMWRITE_JPEG_QUALITY, + self.config.jpeg_quality, + cv2.IMWRITE_PNG_COMPRESSION, + self.config.png_compression, + ], + )[1].tobytes() + data = str(f"data:image/{self.config.image_format};base64," + base64.b64encode(data).decode("ascii")) self.vis["render_img"].write(data) def _update_viewer_stats(self, render_time: float, num_rays: int, image_height: int, image_width: int) -> None: From 3ec8c06e8360ea21e84332c13ec31475cb455269 Mon Sep 17 00:00:00 2001 From: Matt Tancik Date: Wed, 22 Mar 2023 14:37:05 -0700 Subject: [PATCH 6/8] Cleanup --- nerfstudio/model_components/losses.py | 2 +- nerfstudio/model_components/ray_samplers.py | 5 +++++ nerfstudio/models/base_model.py | 1 + nerfstudio/models/tensorf.py | 3 +++ nerfstudio/models/vanilla_nerf.py | 1 + nerfstudio/pipelines/base_pipeline.py | 2 ++ nerfstudio/process_data/hloc_utils.py | 5 +++++ 7 files changed, 18 insertions(+), 1 deletion(-) diff --git a/nerfstudio/model_components/losses.py b/nerfstudio/model_components/losses.py index 174c131cc..d2db011d3 100644 --- a/nerfstudio/model_components/losses.py +++ b/nerfstudio/model_components/losses.py @@ -338,7 +338,7 @@ class MiDaSMSELoss(nn.Module): def __init__(self, reduction_type: Literal["image", "batch"] = "batch"): super().__init__() - self.reduction_type = reduction_type + self.reduction_type: Literal["image", "batch"] = reduction_type # reduction here is different from the image/batch-based reduction. This is either "mean" or "sum" self.mse_loss = MSELoss(reduction="none") diff --git a/nerfstudio/model_components/ray_samplers.py b/nerfstudio/model_components/ray_samplers.py index d75bfec94..f5a96255b 100644 --- a/nerfstudio/model_components/ray_samplers.py +++ b/nerfstudio/model_components/ray_samplers.py @@ -641,6 +641,7 @@ class NeuSSampler(Sampler): # Start with uniform sampling if ray_samples is None: ray_samples = self.uniform_sampler(ray_bundle, num_samples=self.num_samples) + assert ray_samples is not None total_iters = 0 sorted_index = None @@ -697,6 +698,7 @@ class NeuSSampler(Sampler): """ batch_size = ray_samples.shape[0] prev_sdf, next_sdf = sdf[:, :-1], sdf[:, 1:] + assert ray_samples.deltas is not None deltas = ray_samples.deltas[:, :-1, 0] mid_sdf = (prev_sdf + next_sdf) * 0.5 cos_val = (next_sdf - prev_sdf) / (deltas + 1e-5) @@ -738,6 +740,9 @@ class NeuSSampler(Sampler): ray_samples_2 : ray_samples to merge """ + assert ray_samples_1.spacing_starts is not None and ray_samples_2.spacing_starts is not None + assert ray_samples_1.spacing_ends is not None and ray_samples_2.spacing_ends is not None + assert ray_samples_1.spacing_to_euclidean_fn is not None starts_1 = ray_samples_1.spacing_starts[..., 0] starts_2 = ray_samples_2.spacing_starts[..., 0] diff --git a/nerfstudio/models/base_model.py b/nerfstudio/models/base_model.py index 3cc5fa2ec..a73ea67a1 100644 --- a/nerfstudio/models/base_model.py +++ b/nerfstudio/models/base_model.py @@ -101,6 +101,7 @@ class Model(nn.Module): # NOTE: call `super().populate_modules()` in subclasses if self.config.enable_collider: + assert self.config.collider_params is not None self.collider = NearFarCollider( near_plane=self.config.collider_params["near_plane"], far_plane=self.config.collider_params["far_plane"] ) diff --git a/nerfstudio/models/tensorf.py b/nerfstudio/models/tensorf.py index 2da35f5d1..0a988b111 100644 --- a/nerfstudio/models/tensorf.py +++ b/nerfstudio/models/tensorf.py @@ -90,6 +90,8 @@ class TensoRFModel(Model): config: TensoRF configuration to instantiate model """ + config: TensoRFModelConfig + def __init__( self, config: TensoRFModelConfig, @@ -303,6 +305,7 @@ class TensoRFModel(Model): image = batch["image"].to(outputs["rgb"].device) rgb = outputs["rgb"] acc = colormaps.apply_colormap(outputs["accumulation"]) + assert self.config.collider_params is not None depth = colormaps.apply_depth_colormap( outputs["depth"], accumulation=outputs["accumulation"], diff --git a/nerfstudio/models/vanilla_nerf.py b/nerfstudio/models/vanilla_nerf.py index 53e838891..b69a35358 100644 --- a/nerfstudio/models/vanilla_nerf.py +++ b/nerfstudio/models/vanilla_nerf.py @@ -201,6 +201,7 @@ class NeRFModel(Model): rgb_fine = outputs["rgb_fine"] acc_coarse = colormaps.apply_colormap(outputs["accumulation_coarse"]) acc_fine = colormaps.apply_colormap(outputs["accumulation_fine"]) + assert self.config.collider_params is not None depth_coarse = colormaps.apply_depth_colormap( outputs["depth_coarse"], accumulation=outputs["accumulation_coarse"], diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py index bb4a25f95..1da017ebc 100644 --- a/nerfstudio/pipelines/base_pipeline.py +++ b/nerfstudio/pipelines/base_pipeline.py @@ -95,6 +95,7 @@ class Pipeline(nn.Module): datamanager: DataManager _model: Model + world_size: int @property def model(self): @@ -201,6 +202,7 @@ class VanillaPipelineConfig(cfg.InstantiateConfig): class VanillaPipeline(Pipeline): """The pipeline class for the vanilla nerf setup of multiple cameras for one or a few scenes. + Args: config: configuration to instantiate pipeline device: location to place model and data test_mode: diff --git a/nerfstudio/process_data/hloc_utils.py b/nerfstudio/process_data/hloc_utils.py index 0aec2376e..6ec71edd0 100644 --- a/nerfstudio/process_data/hloc_utils.py +++ b/nerfstudio/process_data/hloc_utils.py @@ -75,6 +75,11 @@ def run_hloc( camera_model: Camera model to use. gpu: If True, use GPU. verbose: If True, logs the output of the command. + matching_method: Method to use for matching images. + feature_type: Type of visual features to use. + matcher_type: Type of feature matcher to use. + num_matched: Number of image pairs for loc. + refine_pixsfm: If True, refine the reconstruction using pixel-perfect-sfm. """ if not _HAS_HLOC: CONSOLE.print( From 9b99802042386b5ae2844b1ddccb366ce2a2f24d Mon Sep 17 00:00:00 2001 From: Alvin Xu <66745848+alvin-xu-5745@users.noreply.github.com> Date: Wed, 22 Mar 2023 14:37:35 -0700 Subject: [PATCH 7/8] Check port before loading data (#1633) * Check port before loading data * Fix formatting --- nerfstudio/engine/trainer.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index 208022b8c..0262703dd 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -138,6 +138,18 @@ class Trainer: 'test': loads train/test datasets into memory 'inference': does not load any dataset into memory """ + # set up viewer if enabled + viewer_log_path = self.base_dir / self.config.viewer.relative_log_filename + self.viewer_state, banner_messages = None, None + if self.config.is_viewer_enabled() and self.local_rank == 0: + datapath = self.config.data + if datapath is None: + datapath = self.base_dir + self.viewer_state, banner_messages = viewer_utils.setup_viewer( + self.config.viewer, log_filename=viewer_log_path, datapath=datapath + ) + self._check_viewer_warnings() + self.pipeline = self.config.pipeline.setup( device=self.device, test_mode=test_mode, world_size=self.world_size, local_rank=self.local_rank ) @@ -153,17 +165,6 @@ class Trainer: ) ) - # set up viewer if enabled - viewer_log_path = self.base_dir / self.config.viewer.relative_log_filename - self.viewer_state, banner_messages = None, None - if self.config.is_viewer_enabled() and self.local_rank == 0: - datapath = self.pipeline.datamanager.get_datapath() - if datapath is None: - datapath = self.base_dir - self.viewer_state, banner_messages = viewer_utils.setup_viewer( - self.config.viewer, log_filename=viewer_log_path, datapath=datapath - ) - self._check_viewer_warnings() # set up writers/profilers if enabled writer_log_path = self.base_dir / self.config.logging.relative_log_dir writer.setup_event_writer( From 2c79d9fe52ae2eef2887aa621ae701ef91da95b0 Mon Sep 17 00:00:00 2001 From: machenmusik Date: Wed, 22 Mar 2023 22:03:23 -0400 Subject: [PATCH 8/8] realitycapture: allow periods within filename, not only for file extension (#1649) allow periods within filename, not only for file extension --- nerfstudio/process_data/realitycapture_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nerfstudio/process_data/realitycapture_utils.py b/nerfstudio/process_data/realitycapture_utils.py index 7bc73e132..87ce9240c 100644 --- a/nerfstudio/process_data/realitycapture_utils.py +++ b/nerfstudio/process_data/realitycapture_utils.py @@ -60,7 +60,7 @@ def realitycapture_to_json( cameras.setdefault(column, []).append(value) for name in cameras["#name"]: - camera_label = name.split(".")[0] + camera_label = name.rpartition(".")[0] if camera_label in image_filename_map: img = np.array(Image.open(output_dir / image_filename_map[camera_label])) break @@ -73,7 +73,7 @@ def realitycapture_to_json( missing_image_data = 0 for i, name in enumerate(cameras["#name"]): - basename = name.split(".")[0] + basename = name.rpartition(".")[0] if basename not in image_filename_map: if verbose: CONSOLE.print(f"Missing image for camera data {basename}, Skipping")