diff --git a/nerfstudio/scripts/exporter.py b/nerfstudio/scripts/exporter.py index 84ee4c5b6..c359efcaf 100644 --- a/nerfstudio/scripts/exporter.py +++ b/nerfstudio/scripts/exporter.py @@ -61,9 +61,7 @@ class Exporter: """Set to True when export is finished.""" -def validate_pipeline( - normal_method: str, normal_output_name: str, pipeline: Pipeline -) -> None: +def validate_pipeline(normal_method: str, normal_output_name: str, pipeline: Pipeline) -> None: """Check that the pipeline is valid for this exporter. Args: @@ -85,9 +83,7 @@ def validate_pipeline( ) outputs = pipeline.model(ray_bundle) if normal_output_name not in outputs: - CONSOLE.print( - f"[bold yellow]Warning: Normal output '{normal_output_name}' not found in pipeline outputs." - ) + CONSOLE.print(f"[bold yellow]Warning: Normal output '{normal_output_name}' not found in pipeline outputs.") CONSOLE.print(f"Available outputs: {list(outputs.keys())}") CONSOLE.print( "[bold yellow]Warning: Please train a model with normals " @@ -152,21 +148,13 @@ class ExportPointCloud(Exporter): ), ) assert pipeline.datamanager.train_pixel_sampler is not None - pipeline.datamanager.train_pixel_sampler.num_rays_per_batch = ( - self.num_rays_per_batch - ) + pipeline.datamanager.train_pixel_sampler.num_rays_per_batch = self.num_rays_per_batch # Whether the normals should be estimated based on the point cloud. estimate_normals = self.normal_method == "open3d" crop_obb = None - if ( - self.obb_center is not None - and self.obb_rotation is not None - and self.obb_scale is not None - ): - crop_obb = OrientedBox.from_params( - self.obb_center, self.obb_rotation, self.obb_scale - ) + if self.obb_center is not None and self.obb_rotation is not None and self.obb_scale is not None: + crop_obb = OrientedBox.from_params(self.obb_center, self.obb_rotation, self.obb_scale) pcd = generate_point_cloud( pipeline=pipeline, num_points=self.num_points, @@ -175,20 +163,14 @@ class ExportPointCloud(Exporter): estimate_normals=estimate_normals, rgb_output_name=self.rgb_output_name, depth_output_name=self.depth_output_name, - normal_output_name=( - self.normal_output_name - if self.normal_method == "model_output" - else None - ), + normal_output_name=(self.normal_output_name if self.normal_method == "model_output" else None), crop_obb=crop_obb, std_ratio=self.std_ratio, ) if self.save_world_frame: # apply the inverse dataparser transform to the point cloud points = np.asarray(pcd.points) - poses = np.eye(4, dtype=np.float32)[None, ...].repeat( - points.shape[0], axis=0 - )[:, :3, :] + poses = np.eye(4, dtype=np.float32)[None, ...].repeat(points.shape[0], axis=0)[:, :3, :] poses[:, :3, 3] = points poses = pipeline.datamanager.train_dataparser_outputs.transform_poses_to_original_space( torch.from_numpy(poses) @@ -279,9 +261,7 @@ class ExportTSDFMesh(Exporter): mesh, pipeline, self.output_dir, - px_per_uv_triangle=( - self.px_per_uv_triangle if self.unwrap_method == "custom" else None - ), + px_per_uv_triangle=(self.px_per_uv_triangle if self.unwrap_method == "custom" else None), unwrap_method=self.unwrap_method, num_pixels_per_side=self.num_pixels_per_side, ) @@ -359,20 +339,12 @@ class ExportPoissonMesh(Exporter): ), ) assert pipeline.datamanager.train_pixel_sampler is not None - pipeline.datamanager.train_pixel_sampler.num_rays_per_batch = ( - self.num_rays_per_batch - ) + pipeline.datamanager.train_pixel_sampler.num_rays_per_batch = self.num_rays_per_batch # Whether the normals should be estimated based on the point cloud. estimate_normals = self.normal_method == "open3d" - if ( - self.obb_center is not None - and self.obb_rotation is not None - and self.obb_scale is not None - ): - crop_obb = OrientedBox.from_params( - self.obb_center, self.obb_rotation, self.obb_scale - ) + if self.obb_center is not None and self.obb_rotation is not None and self.obb_scale is not None: + crop_obb = OrientedBox.from_params(self.obb_center, self.obb_rotation, self.obb_scale) else: crop_obb = None @@ -384,11 +356,7 @@ class ExportPoissonMesh(Exporter): estimate_normals=estimate_normals, rgb_output_name=self.rgb_output_name, depth_output_name=self.depth_output_name, - normal_output_name=( - self.normal_output_name - if self.normal_method == "model_output" - else None - ), + normal_output_name=(self.normal_output_name if self.normal_method == "model_output" else None), crop_obb=crop_obb, std_ratio=self.std_ratio, ) @@ -402,9 +370,7 @@ class ExportPoissonMesh(Exporter): CONSOLE.print("[bold green]:white_check_mark: Saving Point Cloud") CONSOLE.print("Computing Mesh... this may take a while.") - mesh, densities = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson( - pcd, depth=9 - ) + mesh, densities = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(pcd, depth=9) vertices_to_remove = densities < np.quantile(densities, 0.1) mesh.remove_vertices_by_mask(vertices_to_remove) print("\033[A\033[A") @@ -428,9 +394,7 @@ class ExportPoissonMesh(Exporter): mesh, pipeline, self.output_dir, - px_per_uv_triangle=( - self.px_per_uv_triangle if self.unwrap_method == "custom" else None - ), + px_per_uv_triangle=(self.px_per_uv_triangle if self.unwrap_method == "custom" else None), unwrap_method=self.unwrap_method, num_pixels_per_side=self.num_pixels_per_side, ) @@ -469,15 +433,11 @@ class ExportMarchingCubesMesh(Exporter): _, pipeline, _, _ = eval_setup(self.load_config) # TODO: Make this work with Density Field - assert hasattr( - pipeline.model.config, "sdf_field" - ), "Model must have an SDF field." + assert hasattr(pipeline.model.config, "sdf_field"), "Model must have an SDF field." CONSOLE.print("Extracting mesh with marching cubes... which may take a while") - assert ( - self.resolution % 512 == 0 - ), f"""resolution must be divisible by 512, got {self.resolution}. + assert self.resolution % 512 == 0, f"""resolution must be divisible by 512, got {self.resolution}. This is important because the algorithm uses a multi-resolution approach to evaluate the SDF where the minimum resolution is 512.""" @@ -496,17 +456,13 @@ class ExportMarchingCubesMesh(Exporter): multi_res_mesh.export(filename) # load the mesh from the marching cubes export - mesh = get_mesh_from_filename( - str(filename), target_num_faces=self.target_num_faces - ) + mesh = get_mesh_from_filename(str(filename), target_num_faces=self.target_num_faces) CONSOLE.print("Texturing mesh with NeRF...") texture_utils.export_textured_mesh( mesh, pipeline, self.output_dir, - px_per_uv_triangle=( - self.px_per_uv_triangle if self.unwrap_method == "custom" else None - ), + px_per_uv_triangle=(self.px_per_uv_triangle if self.unwrap_method == "custom" else None), unwrap_method=self.unwrap_method, num_pixels_per_side=self.num_pixels_per_side, ) @@ -534,9 +490,7 @@ class ExportCameraPoses(Exporter): ("transforms_eval.json", eval_frames), ]: if len(frames) == 0: - CONSOLE.print( - f"[bold yellow]No frames found for {file_name}. Skipping." - ) + CONSOLE.print(f"[bold yellow]No frames found for {file_name}. Skipping.") continue output_file_path = os.path.join(self.output_dir, file_name) @@ -544,9 +498,7 @@ class ExportCameraPoses(Exporter): with open(output_file_path, "w", encoding="UTF-8") as f: json.dump(frames, f, indent=4) - CONSOLE.print( - f"[bold green]:white_check_mark: Saved poses to {output_file_path}" - ) + CONSOLE.print(f"[bold green]:white_check_mark: Saved poses to {output_file_path}") @dataclass @@ -590,9 +542,7 @@ class ExportGaussianSplat(Exporter): and tensor.size > 0 for tensor in map_to_tensors.values() ): - raise ValueError( - "All tensors must be numpy arrays of float or uint8 type and not empty" - ) + raise ValueError("All tensors must be numpy arrays of float or uint8 type and not empty") with open(filename, "wb") as ply_file: # Write PLY header @@ -668,14 +618,8 @@ class ExportGaussianSplat(Exporter): for i in range(4): map_to_tensors[f"rot_{i}"] = quats[:, i, None] - if ( - self.obb_center is not None - and self.obb_rotation is not None - and self.obb_scale is not None - ): - crop_obb = OrientedBox.from_params( - self.obb_center, self.obb_rotation, self.obb_scale - ) + if self.obb_center is not None and self.obb_rotation is not None and self.obb_scale is not None: + crop_obb = OrientedBox.from_params(self.obb_center, self.obb_rotation, self.obb_scale) assert crop_obb is not None mask = crop_obb.within(torch.from_numpy(positions)).numpy() for k, t in map_to_tensors.items(): @@ -695,9 +639,7 @@ class ExportGaussianSplat(Exporter): CONSOLE.print(f"{n_before - n_after} NaN/Inf elements in {k}") if np.sum(select) < n: - CONSOLE.print( - f"values have NaN/Inf in map_to_tensors, only export {np.sum(select)}/{n}" - ) + CONSOLE.print(f"values have NaN/Inf in map_to_tensors, only export {np.sum(select)}/{n}") for k, t in map_to_tensors.items(): map_to_tensors[k] = map_to_tensors[k][select] count = np.sum(select) diff --git a/nerfstudio/scripts/render.py b/nerfstudio/scripts/render.py index 9668e0171..d71bb7f23 100644 --- a/nerfstudio/scripts/render.py +++ b/nerfstudio/scripts/render.py @@ -153,19 +153,14 @@ def _render_trajectory_video( assert train_dataset is not None assert train_cameras is not None cam_pos = cameras[camera_idx].camera_to_worlds[:, 3].cpu() - cam_quat = tf.SO3.from_matrix( - cameras[camera_idx].camera_to_worlds[:3, :3].numpy(force=True) - ).wxyz + cam_quat = tf.SO3.from_matrix(cameras[camera_idx].camera_to_worlds[:3, :3].numpy(force=True)).wxyz for i in range(len(train_cameras)): train_cam_pos = train_cameras[i].camera_to_worlds[:, 3].cpu() # Make sure the line of sight from rendered cam to training cam is not blocked by any object bundle = RayBundle( origins=cam_pos.view(1, 3), - directions=( - (cam_pos - train_cam_pos) - / (cam_pos - train_cam_pos).norm() - ).view(1, 3), + directions=((cam_pos - train_cam_pos) / (cam_pos - train_cam_pos).norm()).view(1, 3), pixel_area=torch.tensor(1).view(1, 1), nears=torch.tensor(0.05).view(1, 1), fars=torch.tensor(100).view(1, 1), @@ -174,9 +169,7 @@ def _render_trajectory_video( ).to(pipeline.device) outputs = pipeline.model.get_outputs(bundle) - q = tf.SO3.from_matrix( - train_cameras[i].camera_to_worlds[:3, :3].numpy(force=True) - ).wxyz + q = tf.SO3.from_matrix(train_cameras[i].camera_to_worlds[:3, :3].numpy(force=True)).wxyz # calculate distance between two quaternions rot_dist = 1 - np.dot(q, cam_quat) ** 2 pos_dist = torch.norm(train_cam_pos - cam_pos) @@ -186,10 +179,7 @@ def _render_trajectory_video( true_max_dist = dist true_max_idx = i - if ( - outputs["depth"][0] - < torch.norm(cam_pos - train_cam_pos).item() - ): + if outputs["depth"][0] < torch.norm(cam_pos - train_cam_pos).item(): continue if check_occlusions and (max_dist == -1 or dist < max_dist): @@ -399,11 +389,7 @@ class CropData: background_color: Float[Tensor, "3"] = torch.Tensor([0.0, 0.0, 0.0]) """background color""" - obb: OrientedBox = field( - default_factory=lambda: OrientedBox( - R=torch.eye(3), T=torch.zeros(3), S=torch.ones(3) * 2 - ) - ) + obb: OrientedBox = field(default_factory=lambda: OrientedBox(R=torch.eye(3), T=torch.zeros(3), S=torch.ones(3) * 2)) """Oriented box representing the crop region""" # properties for backwards-compatibility interface @@ -429,18 +415,12 @@ def get_crop_from_json(camera_json: Dict[str, Any]) -> Optional[CropData]: bg_color = camera_json["crop"]["crop_bg_color"] center = camera_json["crop"]["crop_center"] scale = camera_json["crop"]["crop_scale"] - rot = ( - (0.0, 0.0, 0.0) - if "crop_rot" not in camera_json["crop"] - else tuple(camera_json["crop"]["crop_rot"]) - ) + rot = (0.0, 0.0, 0.0) if "crop_rot" not in camera_json["crop"] else tuple(camera_json["crop"]["crop_rot"]) assert len(center) == 3 assert len(scale) == 3 assert len(rot) == 3 return CropData( - background_color=torch.Tensor( - [bg_color["r"] / 255.0, bg_color["g"] / 255.0, bg_color["b"] / 255.0] - ), + background_color=torch.Tensor([bg_color["r"] / 255.0, bg_color["g"] / 255.0, bg_color["b"] / 255.0]), obb=OrientedBox.from_params(center, rot, scale), ) @@ -514,9 +494,7 @@ class RenderCameraPath(BaseRender): or camera_path.camera_type[0] == CameraType.VR180_L.value ): # temp folder for writing left and right view renders - temp_folder_path = self.output_path.parent / ( - self.output_path.stem + "_temp" - ) + temp_folder_path = self.output_path.parent / (self.output_path.stem + "_temp") Path(temp_folder_path).mkdir(parents=True, exist_ok=True) left_eye_path = temp_folder_path / "render_left.mp4" @@ -524,9 +502,7 @@ class RenderCameraPath(BaseRender): self.output_path = left_eye_path if camera_path.camera_type[0] == CameraType.OMNIDIRECTIONALSTEREO_L.value: - CONSOLE.print( - "[bold green]:goggles: Omni-directional Stereo VR :goggles:" - ) + CONSOLE.print("[bold green]:goggles: Omni-directional Stereo VR :goggles:") else: CONSOLE.print("[bold green]:goggles: VR180 :goggles:") @@ -816,39 +792,25 @@ class DatasetRender(BaseRender): update_config_callback=update_config, ) data_manager_config = config.pipeline.datamanager - assert isinstance( - data_manager_config, (VanillaDataManagerConfig, FullImageDatamanagerConfig) - ) + assert isinstance(data_manager_config, (VanillaDataManagerConfig, FullImageDatamanagerConfig)) for split in self.split.split("+"): datamanager: VanillaDataManager dataset: Dataset if split == "train": - with _disable_datamanager_setup( - data_manager_config._target - ): # pylint: disable=protected-access - datamanager = data_manager_config.setup( - test_mode="test", device=pipeline.device - ) + with _disable_datamanager_setup(data_manager_config._target): # pylint: disable=protected-access + datamanager = data_manager_config.setup(test_mode="test", device=pipeline.device) dataset = datamanager.train_dataset - dataparser_outputs = getattr( - dataset, "_dataparser_outputs", datamanager.train_dataparser_outputs - ) + dataparser_outputs = getattr(dataset, "_dataparser_outputs", datamanager.train_dataparser_outputs) else: - with _disable_datamanager_setup( - data_manager_config._target - ): # pylint: disable=protected-access - datamanager = data_manager_config.setup( - test_mode=split, device=pipeline.device - ) + with _disable_datamanager_setup(data_manager_config._target): # pylint: disable=protected-access + datamanager = data_manager_config.setup(test_mode=split, device=pipeline.device) dataset = datamanager.eval_dataset dataparser_outputs = getattr(dataset, "_dataparser_outputs", None) if dataparser_outputs is None: - dataparser_outputs = datamanager.dataparser.get_dataparser_outputs( - split=datamanager.test_split - ) + dataparser_outputs = datamanager.dataparser.get_dataparser_outputs(split=datamanager.test_split) dataloader = FixedIndicesEvalDataloader( input_dataset=dataset, device=datamanager.device, @@ -866,9 +828,7 @@ class DatasetRender(BaseRender): TimeRemainingColumn(elapsed_when_finished=False, compact=False), TimeElapsedColumn(), ) as progress: - for camera_idx, (camera, batch) in enumerate( - progress.track(dataloader, total=len(dataset)) - ): + for camera_idx, (camera, batch) in enumerate(progress.track(dataloader, total=len(dataset))): with torch.no_grad(): outputs = pipeline.model.get_outputs_for_camera(camera) @@ -901,13 +861,9 @@ class DatasetRender(BaseRender): image_name = f"{camera_idx:05d}" # Try to get the original filename - image_name = dataparser_outputs.image_filenames[ - camera_idx - ].relative_to(images_root) + image_name = dataparser_outputs.image_filenames[camera_idx].relative_to(images_root) - output_path = ( - self.output_path / split / rendered_output_name / image_name - ) + output_path = self.output_path / split / rendered_output_name / image_name output_path.parent.mkdir(exist_ok=True, parents=True) output_name = rendered_output_name @@ -921,9 +877,7 @@ class DatasetRender(BaseRender): output_image = outputs[output_name] if is_depth: # Divide by the dataparser scale factor - output_image.div_( - dataparser_outputs.dataparser_scale - ) + output_image.div_(dataparser_outputs.dataparser_scale) else: if output_name.startswith("gt-"): output_name = output_name[3:] @@ -959,14 +913,10 @@ class DatasetRender(BaseRender): # Save to file if is_raw: - with gzip.open( - output_path.with_suffix(".npy.gz"), "wb" - ) as f: + with gzip.open(output_path.with_suffix(".npy.gz"), "wb") as f: np.save(f, output_image) elif self.image_format == "png": - media.write_image( - output_path.with_suffix(".png"), output_image, fmt="png" - ) + media.write_image(output_path.with_suffix(".png"), output_image, fmt="png") elif self.image_format == "jpeg": media.write_image( output_path.with_suffix(".jpg"), @@ -975,9 +925,7 @@ class DatasetRender(BaseRender): quality=self.jpeg_quality, ) else: - raise ValueError( - f"Unknown image format {self.image_format}" - ) + raise ValueError(f"Unknown image format {self.image_format}") table = Table( title=None,