diff --git a/docs/developer_guides/pipelines/dataparsers.md b/docs/developer_guides/pipelines/dataparsers.md index 6b81803c2..3a38664f9 100644 --- a/docs/developer_guides/pipelines/dataparsers.md +++ b/docs/developer_guides/pipelines/dataparsers.md @@ -67,10 +67,16 @@ class NerfstudioDataParserConfig(DataParserConfig): """How much to downscale images. If not set, images are chosen such that the max dimension is <1600px.""" scene_scale: float = 1.0 """How much to scale the region of interest by.""" - orientation_method: Literal["pca", "up"] = "up" + orientation_method: Literal["pca", "up", "vertical", "none"] = "up" """The method to use for orientation.""" + center_method: Literal["poses", "focus", "none"] = "poses" + """The method to use to center the poses.""" + auto_scale_poses: bool = True + """Whether to automatically scale the poses to fit in +/- 1 bounding box.""" train_split_fraction: float = 0.9 """The fraction of images to use for training. The remaining images are for eval.""" + depth_unit_scale_factor: float = 1e-3 + """Scales the depth values to meters. Default value is 0.001 for a millimeter to meter conversion.""" @dataclass class Nerfstudio(DataParser): diff --git a/docs/developer_guides/viewer/imgs/viewer_dark.png b/docs/developer_guides/viewer/imgs/viewer_dark.png new file mode 100644 index 000000000..d2abe9a10 Binary files /dev/null and b/docs/developer_guides/viewer/imgs/viewer_dark.png differ diff --git a/docs/developer_guides/viewer/imgs/viewer_light.png b/docs/developer_guides/viewer/imgs/viewer_light.png new file mode 100644 index 000000000..2220e3a8c Binary files /dev/null and b/docs/developer_guides/viewer/imgs/viewer_light.png differ diff --git a/docs/developer_guides/viewer/viewer_overview.md b/docs/developer_guides/viewer/viewer_overview.md index 4e10319b6..55dbab2f2 100644 --- a/docs/developer_guides/viewer/viewer_overview.md +++ b/docs/developer_guides/viewer/viewer_overview.md @@ -4,7 +4,15 @@ The viewer is built using [ThreeJS](https://threejs.org/) and packaged into a [ReactJS](https://reactjs.org/) application. This client viewer application will connect via a websocket to a server running on your machine. The following figure helps to illustrate how our viewer framework works: -![visualize_dataset](imgs/viewer_figure.png) +```{image} imgs/viewer_light.png +:align: center +:class: only-light +``` + +```{image} imgs/viewer_dark.png +:align: center +:class: only-dark +``` ## System design @@ -14,12 +22,10 @@ In the center, we have the Bridge Server, which facilitates the connection betwe #### Connection between the Bridge Server and the Client App -The connection between the Bridge Server and the Client App works with WebSockets and WebRTC. +The connection between the Bridge Server and the Client App works with WebSockets. - **WebSocket connection** - The WebSocket is used by the Bridge Server to dispatch commands coming from the nerfstudio TCP connection. Commands can be used for drawing primitives, for setting the transform of objects, for the setting various properties, and more. -- **WebRTC connection** - We use WebRTC to stream images being rendered from nerfstudio. The websocket connection if first used to establish the WebRTC connection. Then, the Client App constantly publishes camera pose information to the Bridge Server and stores the camera information (intrinsics and extrinsics). This information is then queried from the nerfstudio code, used to render an image with some Graph, and then the image is send over the TCP connection and dispatched via WebRTC to render the stream of images. - ## Installing and running locally #### Running the Client App @@ -50,7 +56,7 @@ yarn install yarn start ``` -The local webserver runs on port 4000 by default, +The local webserver runs on port 4000 by default, so when `ns-train` is running, you can connect to the viewer locally at [http://localhost:4000/?websocket_url=ws://localhost:7007](http://localhost:4000/?websocket_url=ws://localhost:7007) @@ -70,7 +76,6 @@ We thank the authors and contributors to the following repos, which we've starte - [meshcat](https://github.com/rdeits/meshcat) - made by [Robin Deits](https://github.com/rdeits) - [ThreeJS](https://threejs.org/) - [ReactJS](https://reactjs.org/) -- [WebRTC](https://webrtc.org/) - WebRTC is a framework for real-time communication that allows two peers to send video, audio, or general data to and from each other with low latency. We've adopted WebRTC to stream rendered images to our viewer. ## FAQ diff --git a/docs/nerfology/methods/nerf.md b/docs/nerfology/methods/nerf.md index 90e89cd59..4c56f3154 100644 --- a/docs/nerfology/methods/nerf.md +++ b/docs/nerfology/methods/nerf.md @@ -18,7 +18,7 @@ ns-train vanilla-nerf If you have arrived to this site, it is likely that you have at least heard of NeRFs. This page will discuss the original NeRF paper, _"NeRF: Representing Scenes as Neural Radiance Fields for View Synthesis"_ by Mildenhall, Srinivasan, Tancik et al. (2020). -For most tasks, using the original NeRF model is likely not a good choice and hence we provide implementations of various other NeRF related models. It is however useful to understand how NeRF's work as most follow ups follow a similar structure. +For most tasks, using the original NeRF model is likely not a good choice and hence we provide implementations of various other NeRF related models. It is however useful to understand how NeRF's work as most follow ups follow a similar structure and it doesn't require CUDA to execute (useful for stepping through the code with a debugger if you don't have a GPU at hand). The goal is to optimize a volumetric representation of a scene that can be rendered from novel viewpoints. This representation is optimized from a set of images and associated camera poses. diff --git a/docs/nerfology/model_components/visualize_samplers.ipynb b/docs/nerfology/model_components/visualize_samplers.ipynb index 1ead09832..a23500d58 100644 --- a/docs/nerfology/model_components/visualize_samplers.ipynb +++ b/docs/nerfology/model_components/visualize_samplers.ipynb @@ -1,6 +1,7 @@ { "cells": [ { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -46,7 +47,7 @@ ":class: only-dark\n", "```\n", "\n", - "### Heirarchical Sampling\n", + "### Hierarchical Sampling\n", "\n", "It is important to sample the scene where it has content otherwise the reconstruction quality will be reduced. \n", "\n", @@ -7413,7 +7414,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.8.12 ('nerfactory')", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -7427,11 +7428,11 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.12" + "version": "3.10.7" }, "vscode": { "interpreter": { - "hash": "34c28001ff35fb390494047002768a8182dcf55b1b11415165e62ea61557ab83" + "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1" } } }, diff --git a/docs/quickstart/custom_dataset.md b/docs/quickstart/custom_dataset.md index ebc6ce313..009cb287c 100644 --- a/docs/quickstart/custom_dataset.md +++ b/docs/quickstart/custom_dataset.md @@ -290,3 +290,35 @@ ns-process-data metashape --data {data directory} --xml {xml file} --output-dir ```bash ns-train nerfacto --data {output directory} ``` + +## Equirectangular data + +Equirectangular data is data that has been taken by a 360 camera such as Insta360. Both equirectangular image sets and videos can be processed by nerfstudio. + +### Images +For a set of equirectangular images, process the data using the following command: + +```bash +ns-process-data images --camera-type equirectangular --images-per-equirect {8, or 14} --data {data directory} --output-dir {output directory} +``` + +The images-per-equirect argument is the number of images that will be sampled from each equirectangular image. We have found that 8 images per equirectangular image is sufficient for most use cases so it defaults to that. However, if you find that there isn't enough detail in the nerf or that colmap is having trouble aligning the images, you can try increasing the number of images per equirectangular image to 14. + +### Videos +For videos we reccomend taking a video with the camera held on top of your head. This will result in any unwanted capturer to just be in the bottom of each frame image and therefore can be cropped out. + +For a video, process the data using the following command: + +```bash +ns-process-data video --camera-type equirectangular --images-per-equirect {8, or 14} --num-frames-target {num equirectangular frames to sample from} --crop-factor {top bottom left right} --data {data directory} --output-dir {output directory} +``` +See the equirectangular images section above for a description of the `--images-per-equirect` argument. + +The `num-frames-target` argument is optional but it is recomended to set it to 3*(seconds of video) frames. For example, if you have a 30 second video, you would use `--num-frames-target 90` (3*30=90). This number was chosen from a bit of experimentation and seems to work well for most videos. It is by no means a hard rule and you can experiment with different values. + +The `crop-factor` argument is optional but often very helpful. This is because equirectangular videos taken by 360 cameras tend to have a portion of the bottom of the image that is the person who was holding the camera over their head. + + + +This obscene mesh of human is obviously not helpful in training a nerf so we can remove it by cropping the bottom 20% of the image. This can be done by using the `--crop-factor 0 0.2 0 0` argument. + diff --git a/docs/quickstart/imgs/equirect_crop.jpg b/docs/quickstart/imgs/equirect_crop.jpg new file mode 100644 index 000000000..584dab831 Binary files /dev/null and b/docs/quickstart/imgs/equirect_crop.jpg differ diff --git a/docs/quickstart/installation.md b/docs/quickstart/installation.md index 2a2db4453..950aefb2a 100644 --- a/docs/quickstart/installation.md +++ b/docs/quickstart/installation.md @@ -140,7 +140,7 @@ To restrict to only CUDA architectures that you have available locally, use the build arg and look up [the compute capability for your GPU](https://developer.nvidia.com/cuda-gpus). For example, here's how to build with support for GeForce 30xx series GPUs: ```bash -docker build --build-arg CUDA_ARCHITECTURES=86 -tag nerfstudio-86 . +docker build --build-arg CUDA_ARCHITECTURES=86 --tag nerfstudio-86 -f Dockerfile . ``` ### Using an interactive container diff --git a/nerfstudio/cameras/camera_utils.py b/nerfstudio/cameras/camera_utils.py index b1dd5a267..d2da627b7 100644 --- a/nerfstudio/cameras/camera_utils.py +++ b/nerfstudio/cameras/camera_utils.py @@ -229,7 +229,7 @@ def get_interpolated_poses_many( traj = np.stack(traj, axis=0) k_interp = np.stack(k_interp, axis=0) - return torch.tensor(traj), torch.tensor(k_interp) + return torch.tensor(traj, dtype=torch.float32), torch.tensor(k_interp, dtype=torch.float32) def normalize(x: torch.Tensor) -> TensorType[...]: @@ -429,35 +429,89 @@ def rotation_matrix(a: TensorType[3], b: TensorType[3]) -> TensorType[3, 3]: return torch.eye(3) + skew_sym_mat + skew_sym_mat @ skew_sym_mat * ((1 - c) / (s**2 + 1e-8)) +def focus_of_attention(poses: TensorType["num_poses":..., 4, 4], initial_focus: TensorType[3]) -> TensorType[3]: + """Compute the focus of attention of a set of cameras. Only cameras + that have the focus of attention in front of them are considered. + + Args: + poses: The poses to orient. + initial_focus: The 3D point views to decide which cameras are initially activated. + + Returns: + The 3D position of the focus of attention. + """ + # References to the same method in third-party code: + # https://github.com/google-research/multinerf/blob/1c8b1c552133cdb2de1c1f3c871b2813f6662265/internal/camera_utils.py#L145 + # https://github.com/bmild/nerf/blob/18b8aebda6700ed659cb27a0c348b737a5f6ab60/load_llff.py#L197 + active_directions = -poses[:, :3, 2:3] + active_origins = poses[:, :3, 3:4] + # initial value for testing if the focus_pt is in front or behind + focus_pt = initial_focus + # Prune cameras which have the current have the focus_pt behind them. + active = torch.sum(active_directions.squeeze(-1) * (focus_pt - active_origins.squeeze(-1)), dim=-1) > 0 + done = False + # We need at least two active cameras, else fallback on the previous solution. + # This may be the "poses" solution if no cameras are active on first iteration, e.g. + # they are in an outward-looking configuration. + while torch.sum(active.int()) > 1 and not done: + active_directions = active_directions[active] + active_origins = active_origins[active] + # https://en.wikipedia.org/wiki/Line–line_intersection#In_more_than_two_dimensions + m = torch.eye(3) - active_directions * torch.transpose(active_directions, -2, -1) + mt_m = torch.transpose(m, -2, -1) @ m + focus_pt = torch.linalg.inv(mt_m.mean(0)) @ (mt_m @ active_origins).mean(0)[:, 0] + active = torch.sum(active_directions.squeeze(-1) * (focus_pt - active_origins.squeeze(-1)), dim=-1) > 0 + if active.all(): + # the set of active cameras did not change, so we're done. + done = True + return focus_pt + + def auto_orient_and_center_poses( - poses: TensorType["num_poses":..., 4, 4], method: Literal["pca", "up", "none"] = "up", center_poses: bool = True + poses: TensorType["num_poses":..., 4, 4], + method: Literal["pca", "up", "vertical", "none"] = "up", + center_method: Literal["poses", "focus", "none"] = "poses", ) -> Tuple[TensorType["num_poses":..., 3, 4], TensorType[4, 4]]: """Orients and centers the poses. We provide two methods for orientation: pca and up. - pca: Orient the poses so that the principal component of the points is aligned with the axes. - This method works well when all of the cameras are in the same plane. + pca: Orient the poses so that the principal directions of the camera centers are aligned + with the axes, Z corresponding to the smallest principal component. + This method works well when all of the cameras are in the same plane, for example when + images are taken using a mobile robot. up: Orient the poses so that the average up vector is aligned with the z axis. This method works well when images are not at arbitrary angles. + vertical: Orient the poses so that the Z 3D direction projects close to the + y axis in images. This method works better if cameras are not all + looking in the same 3D direction, which may happen in camera arrays or in LLFF. + There are two centering methods: + poses: The poses are centered around the origin. + focus: The origin is set to the focus of attention of all cameras (the + closest point to cameras optical axes). Recommended for inward-looking + camera configurations. Args: poses: The poses to orient. method: The method to use for orientation. - center_poses: If True, the poses are centered around the origin. + center_method: The method to use to center the poses. Returns: Tuple of the oriented poses and the transform matrix. """ - translation = poses[..., :3, 3] + origins = poses[..., :3, 3] - mean_translation = torch.mean(translation, dim=0) - translation_diff = translation - mean_translation + mean_origin = torch.mean(origins, dim=0) + translation_diff = origins - mean_origin - if center_poses: - translation = mean_translation + if center_method == "poses": + translation = mean_origin + elif center_method == "focus": + translation = focus_of_attention(poses, mean_origin) + elif center_method == "none": + translation = torch.zeros_like(mean_origin) else: - translation = torch.zeros_like(mean_translation) + raise ValueError(f"Unknown value for center_method: {center_method}") if method == "pca": _, eigvec = torch.linalg.eigh(translation_diff.T @ translation_diff) @@ -471,9 +525,41 @@ def auto_orient_and_center_poses( if oriented_poses.mean(axis=0)[2, 1] < 0: oriented_poses[:, 1:3] = -1 * oriented_poses[:, 1:3] - elif method == "up": + elif method in ("up", "vertical"): up = torch.mean(poses[:, :3, 1], dim=0) up = up / torch.linalg.norm(up) + if method == "vertical": + # If cameras are not all parallel (e.g. not in an LLFF configuration), + # we can find the 3D direction that most projects vertically in all + # cameras by minimizing ||Xu|| s.t. ||u||=1. This total least squares + # problem is solved by SVD. + x_axis_matrix = poses[:, :3, 0] + _, S, Vh = torch.linalg.svd(x_axis_matrix, full_matrices=False) + # Singular values are S_i=||Xv_i|| for each right singular vector v_i. + # ||S|| = sqrt(n) because lines of X are all unit vectors and the v_i + # are an orthonormal basis. + # ||Xv_i|| = sqrt(sum(dot(x_axis_j,v_i)^2)), thus S_i/sqrt(n) is the + # RMS of cosines between x axes and v_i. If the second smallest singular + # value corresponds to an angle error less than 10° (cos(80°)=0.17), + # this is probably a degenerate camera configuration (typical values + # are around 5° average error for the true vertical). In this case, + # rather than taking the vector corresponding to the smallest singular + # value, we project the "up" vector on the plane spanned by the two + # best singular vectors. We could also just fallback to the "up" + # solution. + if S[1] > 0.17 * math.sqrt(poses.shape[0]): + # regular non-degenerate configuration + up_vertical = Vh[2, :] + # It may be pointing up or down. Use "up" to disambiguate the sign. + up = up_vertical if torch.dot(up_vertical, up) > 0 else -up_vertical + else: + # Degenerate configuration: project "up" on the plane spanned by + # the last two right singular vectors (which are orthogonal to the + # first). v_0 is a unit vector, no need to divide by its norm when + # projecting. + up = up - Vh[0, :] * torch.dot(up, Vh[0, :]) + # re-normalize + up = up / torch.linalg.norm(up) rotation = rotation_matrix(up, torch.Tensor([0, 0, 1])) transform = torch.cat([rotation, rotation @ -translation[..., None]], dim=-1) @@ -483,5 +569,7 @@ def auto_orient_and_center_poses( transform[:3, 3] = -translation transform = transform[:3, :] oriented_poses = transform @ poses + else: + raise ValueError(f"Unknown value for method: {method}") return oriented_poses, transform diff --git a/nerfstudio/configs/base_config.py b/nerfstudio/configs/base_config.py index a2e695c14..9d95cc269 100644 --- a/nerfstudio/configs/base_config.py +++ b/nerfstudio/configs/base_config.py @@ -23,8 +23,6 @@ from dataclasses import dataclass from pathlib import Path from typing import Any, List, Optional, Tuple, Type -from typing_extensions import Literal - # model instances from nerfstudio.utils import writer @@ -148,10 +146,3 @@ class ViewerConfig(PrintableConfig): actually used in training/evaluation. If -1, display all.""" quit_on_train_completion: bool = False """Whether to kill the training job when it has completed. Note this will stop rendering in the viewer.""" - skip_openrelay: bool = False - """Avoid using openrelay to communicate with the viewer. Try disabling if you have trouble - connecting to the viewer""" - codec: Literal["H264", "VP8"] = "VP8" - """Video codec that viewer will use.""" - local: bool = False - """If running local server instance, avoid using relays to communicate with the viewer.""" diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index dd0109c1d..7ff2f9aaf 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -235,24 +235,39 @@ class DataManager(nn.Module): """Sets up the data manager for evaluation""" @abstractmethod - def next_train(self, step: int) -> Tuple: + def next_train(self, step: int) -> Tuple[RayBundle, Dict]: """Returns the next batch of data from the train data manager. - This will be a tuple of all the information that this data manager outputs. + Args: + step: the step number of the eval image to retrieve + Returns: + A tuple of the ray bundle for the image, and a dictionary of additional batch information + such as the groudtruth image. """ raise NotImplementedError @abstractmethod - def next_eval(self, step: int) -> Tuple: + def next_eval(self, step: int) -> Tuple[RayBundle, Dict]: """Returns the next batch of data from the eval data manager. - This will be a tuple of all the information that this data manager outputs. + Args: + step: the step number of the eval image to retrieve + Returns: + A tuple of the ray bundle for the image, and a dictionary of additional batch information + such as the groudtruth image. """ raise NotImplementedError @abstractmethod - def next_eval_image(self, step: int) -> Tuple: - """Returns the next eval image.""" + def next_eval_image(self, step: int) -> Tuple[int, RayBundle, Dict]: + """Retreive the next eval image. + + Args: + step: the step number of the eval image to retrieve + Returns: + A tuple of the step number, the ray bundle for the image, and a dictionary of + additional batch information such as the groudtruth image. + """ raise NotImplementedError @abstractmethod @@ -358,12 +373,12 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method self.sampler = None self.test_mode = test_mode self.test_split = "test" if test_mode in ["test", "inference"] else "val" - self.dataparser = self.config.dataparser + self.dataparser_config = self.config.dataparser if self.config.data is not None: self.config.dataparser.data = Path(self.config.data) else: self.config.data = self.config.dataparser.data - self.dataparser = self.dataparser.setup() + self.dataparser = self.dataparser_config.setup() self.train_dataparser_outputs = self.dataparser.get_dataparser_outputs(split="train") self.train_dataset = self.create_train_dataset() @@ -453,7 +468,6 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method ) self.eval_dataloader = RandIndicesEvalDataloader( input_dataset=self.eval_dataset, - image_indices=self.config.eval_image_indices, device=self.device, num_workers=self.world_size * 4, ) diff --git a/nerfstudio/data/dataparsers/arkitscenes_dataparser.py b/nerfstudio/data/dataparsers/arkitscenes_dataparser.py index 736f9941b..a92745bc6 100644 --- a/nerfstudio/data/dataparsers/arkitscenes_dataparser.py +++ b/nerfstudio/data/dataparsers/arkitscenes_dataparser.py @@ -21,7 +21,9 @@ from typing import Type import cv2 import numpy as np import torch +from typing_extensions import Literal +from nerfstudio.cameras import camera_utils from nerfstudio.cameras.cameras import Cameras, CameraType from nerfstudio.data.dataparsers.base_dataparser import ( DataParser, @@ -70,11 +72,13 @@ class ARKitScenesDataParserConfig(DataParserConfig): """target class to instantiate""" data: Path = Path("data/ARKitScenes/3dod/Validation/41069021") """Path to ARKitScenes folder with densely extracted scenes.""" + scale_factor: float = 1.0 + """How much to scale the camera origins by.""" scene_scale: float = 1.0 """How much to scale the region of interest by.""" - center_poses: bool = True - """Whether to center the poses.""" - scale_poses: bool = True + center_method: Literal["poses", "focus", "none"] = "poses" + """The method to use to center the poses.""" + auto_scale_poses: bool = True """Whether to automatically scale the poses to fit in +/- 1 bounding box.""" train_split_fraction: float = 0.9 """The fraction of images to use for training. The remaining images are for eval.""" @@ -141,11 +145,19 @@ class ARKitScenes(DataParser): poses = torch.from_numpy(np.stack(poses).astype(np.float32)) intrinsics = torch.from_numpy(np.stack(intrinsics).astype(np.float32)) - if self.config.center_poses: - poses[:, :3, 3] -= poses[:, :3, 3].mean(dim=0) + poses, transform_matrix = camera_utils.auto_orient_and_center_poses( + poses, + method="none", + center_method=self.config.center_method, + ) - if self.config.scale_poses: - poses[:, :3, 3] /= poses[:, :3, 3].abs().max() + # Scale poses + scale_factor = 1.0 + if self.config.auto_scale_poses: + scale_factor /= float(torch.max(torch.abs(poses[:, :3, 3]))) + scale_factor *= self.config.scale_factor + + poses[:, :3, 3] *= scale_factor # Choose image_filenames and poses based on split, but after auto orient and scaling the poses. image_filenames = [image_filenames[i] for i in indices] @@ -177,6 +189,8 @@ class ARKitScenes(DataParser): image_filenames=image_filenames, cameras=cameras, scene_box=scene_box, + dataparser_scale=scale_factor, + dataparser_transform=transform_matrix, metadata={ "depth_filenames": depth_filenames if len(depth_filenames) > 0 else None, "depth_unit_scale_factor": self.config.depth_unit_scale_factor, @@ -208,4 +222,6 @@ class ARKitScenes(DataParser): frame_pose = np.array(poses_from_traj[str(my_key)]) frame_pose[0:3, 1:3] *= -1 + frame_pose = frame_pose[np.array([1, 0, 2, 3]), :] + frame_pose[2, :] *= -1 return frame_pose diff --git a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py index 035e2da2d..adb09eb48 100644 --- a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py +++ b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py @@ -54,10 +54,10 @@ class NerfstudioDataParserConfig(DataParserConfig): """How much to downscale images. If not set, images are chosen such that the max dimension is <1600px.""" scene_scale: float = 1.0 """How much to scale the region of interest by.""" - orientation_method: Literal["pca", "up", "none"] = "up" + orientation_method: Literal["pca", "up", "vertical", "none"] = "up" """The method to use for orientation.""" - center_poses: bool = True - """Whether to center the poses.""" + center_method: Literal["poses", "focus", "none"] = "poses" + """The method to use to center the poses.""" auto_scale_poses: bool = True """Whether to automatically scale the poses to fit in +/- 1 bounding box.""" train_split_fraction: float = 0.9 @@ -209,7 +209,7 @@ class Nerfstudio(DataParser): poses, transform_matrix = camera_utils.auto_orient_and_center_poses( poses, method=orientation_method, - center_poses=self.config.center_poses, + center_method=self.config.center_method, ) # Scale poses diff --git a/nerfstudio/data/dataparsers/phototourism_dataparser.py b/nerfstudio/data/dataparsers/phototourism_dataparser.py index 5b5287d3d..e5a88e10e 100644 --- a/nerfstudio/data/dataparsers/phototourism_dataparser.py +++ b/nerfstudio/data/dataparsers/phototourism_dataparser.py @@ -60,12 +60,12 @@ class PhototourismDataParserConfig(DataParserConfig): """The fraction of images to use for training. The remaining images are for eval.""" scene_scale: float = 1.0 """How much to scale the region of interest by.""" - orientation_method: Literal["pca", "up", "none"] = "up" + orientation_method: Literal["pca", "up", "vertical", "none"] = "up" """The method to use for orientation.""" + center_method: Literal["poses", "focus", "none"] = "poses" + """The method to use to center the poses.""" auto_scale_poses: bool = True """Whether to automatically scale the poses to fit in +/- 1 bounding box.""" - center_poses: bool = True - """Whether to center the poses.""" @dataclass @@ -147,7 +147,7 @@ class Phototourism(DataParser): raise ValueError(f"Unknown dataparser split {split}") poses, transform_matrix = camera_utils.auto_orient_and_center_poses( - poses, method=self.config.orientation_method, center_poses=self.config.center_poses + poses, method=self.config.orientation_method, center_method=self.config.center_method ) # Scale poses diff --git a/nerfstudio/data/dataparsers/scannet_dataparser.py b/nerfstudio/data/dataparsers/scannet_dataparser.py index 80d0dd688..d5a787799 100644 --- a/nerfstudio/data/dataparsers/scannet_dataparser.py +++ b/nerfstudio/data/dataparsers/scannet_dataparser.py @@ -21,7 +21,9 @@ from typing import Type import cv2 import numpy as np import torch +from typing_extensions import Literal +from nerfstudio.cameras import camera_utils from nerfstudio.cameras.cameras import Cameras, CameraType from nerfstudio.data.dataparsers.base_dataparser import ( DataParser, @@ -47,11 +49,13 @@ class ScanNetDataParserConfig(DataParserConfig): """target class to instantiate""" data: Path = Path("data/scannet/scene0423_02") """Path to ScanNet folder with densely extracted scenes.""" + scale_factor: float = 1.0 + """How much to scale the camera origins by.""" scene_scale: float = 1.0 """How much to scale the region of interest by.""" - center_poses: bool = True - """Whether to center the poses.""" - scale_poses: bool = True + center_method: Literal["poses", "focus", "none"] = "poses" + """The method to use to center the poses.""" + auto_scale_poses: bool = True """Whether to automatically scale the poses to fit in +/- 1 bounding box.""" train_split_fraction: float = 0.9 """The fraction of images to use for training. The remaining images are for eval.""" @@ -77,15 +81,14 @@ class ScanNet(DataParser): first_img = cv2.imread(img_dir_sorted[0].as_posix()) h, w, _ = first_img.shape - image_filenames = [] - depth_filenames = [] - intrinsics = [] - poses = [] + image_filenames, depth_filenames, intrinsics, poses = [], [], [], [] K = np.loadtxt(self.config.data / "intrinsic" / "intrinsic_color.txt") for img, depth, pose in zip(img_dir_sorted, depth_dir_sorted, pose_dir_sorted): pose = np.loadtxt(pose) pose[0:3, 1:3] *= -1 + pose = pose[np.array([1, 0, 2, 3]), :] + pose[2, :] *= -1 # We cannot accept files directly, as some of the poses are invalid if np.isinf(pose).any(): @@ -116,11 +119,19 @@ class ScanNet(DataParser): poses = torch.from_numpy(np.stack(poses).astype(np.float32)) intrinsics = torch.from_numpy(np.stack(intrinsics).astype(np.float32)) - if self.config.center_poses: - poses[:, :3, 3] -= poses[:, :3, 3].mean(dim=0) + poses, transform_matrix = camera_utils.auto_orient_and_center_poses( + poses, + method="none", + center_method=self.config.center_method, + ) - if self.config.scale_poses: - poses[:, :3, 3] /= poses[:, :3, 3].abs().max() + # Scale poses + scale_factor = 1.0 + if self.config.auto_scale_poses: + scale_factor /= float(torch.max(torch.abs(poses[:, :3, 3]))) + scale_factor *= self.config.scale_factor + + poses[:, :3, 3] *= scale_factor # Choose image_filenames and poses based on split, but after auto orient and scaling the poses. image_filenames = [image_filenames[i] for i in indices] @@ -152,6 +163,8 @@ class ScanNet(DataParser): image_filenames=image_filenames, cameras=cameras, scene_box=scene_box, + dataparser_scale=scale_factor, + dataparser_transform=transform_matrix, metadata={ "depth_filenames": depth_filenames if len(depth_filenames) > 0 else None, "depth_unit_scale_factor": self.config.depth_unit_scale_factor, diff --git a/nerfstudio/data/dataparsers/sdfstudio_dataparser.py b/nerfstudio/data/dataparsers/sdfstudio_dataparser.py index f25c8f3e7..43781a00b 100644 --- a/nerfstudio/data/dataparsers/sdfstudio_dataparser.py +++ b/nerfstudio/data/dataparsers/sdfstudio_dataparser.py @@ -116,7 +116,7 @@ class SDFStudio(DataParser): camera_to_worlds, transform = camera_utils.auto_orient_and_center_poses( camera_to_worlds, method="up", - center_poses=False, + center_method="none", ) # scene box from meta data diff --git a/nerfstudio/data/pixel_samplers.py b/nerfstudio/data/pixel_samplers.py index 8ac8e183d..244498df8 100644 --- a/nerfstudio/data/pixel_samplers.py +++ b/nerfstudio/data/pixel_samplers.py @@ -261,6 +261,14 @@ class PatchPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods num_rays = (num_rays_per_batch // (self.patch_size**2)) * (self.patch_size**2) super().__init__(num_rays, keep_full_image, **kwargs) + def set_num_rays_per_batch(self, num_rays_per_batch: int): + """Set the number of rays to sample per batch. Overrided to deal with patch-based sampling. + + Args: + num_rays_per_batch: number of rays to sample per batch + """ + self.num_rays_per_batch = (num_rays_per_batch // (self.patch_size**2)) * (self.patch_size**2) + # overrides base method def sample_method( # pylint: disable=no-self-use self, diff --git a/nerfstudio/data/utils/dataloaders.py b/nerfstudio/data/utils/dataloaders.py index c4e155102..23d712d7e 100644 --- a/nerfstudio/data/utils/dataloaders.py +++ b/nerfstudio/data/utils/dataloaders.py @@ -116,7 +116,7 @@ class CacheDataloader(DataLoader): """Returns a collated batch.""" batch_list = self._get_batch_list() collated_batch = self.collate_fn(batch_list) - collated_batch = get_dict_to_torch(collated_batch, device=self.device) + collated_batch = get_dict_to_torch(collated_batch, device=self.device, exclude=["image"]) return collated_batch def __iter__(self): @@ -183,7 +183,7 @@ class EvalDataloader(DataLoader): """ ray_bundle = self.cameras.generate_rays(camera_indices=image_idx, keep_shape=True) batch = self.input_dataset[image_idx] - batch = get_dict_to_torch(batch, device=self.device) + batch = get_dict_to_torch(batch, device=self.device, exclude=["image"]) return ray_bundle, batch diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index a4a09e7b1..97ff9244a 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -243,7 +243,9 @@ class Trainer: writer.put_dict(name="Train Loss Dict", scalar_dict=loss_dict, step=step) writer.put_dict(name="Train Metrics Dict", scalar_dict=metrics_dict, step=step) - self.eval_iteration(step) + # Do not perform evaluation if there are no validation images + if self.pipeline.datamanager.eval_dataset: + self.eval_iteration(step) if step_check(step, self.config.steps_per_save): self.save_checkpoint(step) diff --git a/nerfstudio/exporter/exporter_utils.py b/nerfstudio/exporter/exporter_utils.py index 9f4168517..2f45badab 100644 --- a/nerfstudio/exporter/exporter_utils.py +++ b/nerfstudio/exporter/exporter_utils.py @@ -178,8 +178,8 @@ def generate_point_cloud( rgbs = torch.cat(rgbs, dim=0) pcd = o3d.geometry.PointCloud() - pcd.points = o3d.utility.Vector3dVector(points.float().cpu().numpy()) - pcd.colors = o3d.utility.Vector3dVector(rgbs.float().cpu().numpy()) + pcd.points = o3d.utility.Vector3dVector(points.double().cpu().numpy()) + pcd.colors = o3d.utility.Vector3dVector(rgbs.double().cpu().numpy()) ind = None if remove_outliers: @@ -203,7 +203,7 @@ def generate_point_cloud( if ind is not None: # mask out normals for points that were removed with remove_outliers normals = normals[ind] - pcd.normals = o3d.utility.Vector3dVector(normals.float().cpu().numpy()) + pcd.normals = o3d.utility.Vector3dVector(normals.double().cpu().numpy()) return pcd diff --git a/nerfstudio/fields/instant_ngp_field.py b/nerfstudio/fields/instant_ngp_field.py index 102912ba7..b0f4e97e4 100644 --- a/nerfstudio/fields/instant_ngp_field.py +++ b/nerfstudio/fields/instant_ngp_field.py @@ -19,6 +19,7 @@ Instant-NGP field implementations using tiny-cuda-nn, torch, .... from typing import Dict, Optional, Tuple +import numpy as np import torch from nerfacc import ContractionType, contract from torch.nn.parameter import Parameter @@ -63,6 +64,7 @@ class TCNNInstantNGPField(Field): contraction_type: type of contraction num_levels: number of levels of the hashmap for the base mlp log2_hashmap_size: size of the hashmap for the base mlp + max_res: maximum resolution of the hashmap for the base mlp """ def __init__( @@ -79,6 +81,7 @@ class TCNNInstantNGPField(Field): contraction_type: ContractionType = ContractionType.UN_BOUNDED_SPHERE, num_levels: int = 16, log2_hashmap_size: int = 19, + max_res: int = 2048, ) -> None: super().__init__() @@ -93,7 +96,8 @@ class TCNNInstantNGPField(Field): self.appearance_embedding = Embedding(num_images, appearance_embedding_dim) # TODO: set this properly based on the aabb - per_level_scale = 1.4472692012786865 + base_res: int = 16 + per_level_scale = np.exp((np.log(max_res) - np.log(base_res)) / (num_levels - 1)) self.direction_encoding = tcnn.Encoding( n_input_dims=3, @@ -111,7 +115,7 @@ class TCNNInstantNGPField(Field): "n_levels": num_levels, "n_features_per_level": 2, "log2_hashmap_size": log2_hashmap_size, - "base_resolution": 16, + "base_resolution": base_res, "per_level_scale": per_level_scale, }, network_config={ diff --git a/nerfstudio/models/instant_ngp.py b/nerfstudio/models/instant_ngp.py index ace4b77c6..2bdf85b6a 100644 --- a/nerfstudio/models/instant_ngp.py +++ b/nerfstudio/models/instant_ngp.py @@ -65,6 +65,10 @@ class InstantNGPModelConfig(ModelConfig): """Number of samples in field evaluation.""" grid_resolution: int = 128 """Resolution of the grid used for the field.""" + max_res: int = 2048 + """Maximum resolution of the hashmap for the base mlp.""" + log2_hashmap_size: int = 19 + """Size of the hashmap for the base mlp""" contraction_type: ContractionType = ContractionType.UN_BOUNDED_SPHERE """Contraction type used for spatial deformation of the field.""" cone_angle: float = 0.004 @@ -103,6 +107,8 @@ class NGPModel(Model): contraction_type=self.config.contraction_type, use_appearance_embedding=self.config.use_appearance_embedding, num_images=self.num_train_data, + log2_hashmap_size=self.config.log2_hashmap_size, + max_res=self.config.max_res, ) self.scene_aabb = Parameter(self.scene_box.aabb.flatten(), requires_grad=False) diff --git a/nerfstudio/process_data/equirect_utils.py b/nerfstudio/process_data/equirect_utils.py index 596af58fb..340dd7884 100644 --- a/nerfstudio/process_data/equirect_utils.py +++ b/nerfstudio/process_data/equirect_utils.py @@ -15,13 +15,15 @@ """Helper utils for processing equirectangular data.""" import os +import sys from pathlib import Path -from typing import Tuple +from typing import List, Tuple import cv2 import numpy as np import torch from equilib import Equi2Pers +from rich.console import Console from rich.progress import ( BarColumn, Progress, @@ -32,11 +34,86 @@ from rich.progress import ( from nerfstudio.utils.rich_utils import ItersPerSecColumn +CONSOLE = Console(width=120) + + +def _crop_bottom(bound_arr: list, fov: int, crop_factor: float) -> List[float]: + """Returns a list of vertical bounds with the bottom cropped. + + Args: + bound_arr (list): List of vertical bounds in ascending order. + fov (int): Field of view of the camera. + crop_factor (float): Portion of the image to crop from the bottom. + + Returns: + list: A new list of bounds with the bottom cropped. + """ + degrees_chopped = 180 * crop_factor + new_bottom_start = 90 - degrees_chopped - fov / 2 + for i, el in reversed(list(enumerate(bound_arr))): + if el > new_bottom_start + fov / 2: + bound_arr[i] = None + elif el > new_bottom_start: + diff = el - new_bottom_start + bound_arr[i] = new_bottom_start + for j in range(i - 1, -1, -1): + bound_arr[j] -= diff / (2 ** (i - j)) + break + + return bound_arr + + +def _crop_top(bound_arr: list, fov: int, crop_factor: float) -> List[float]: + """Returns a list of vertical bounds with the top cropped. + + Args: + bound_arr (list): List of vertical bounds in ascending order. + fov (int): Field of view of the camera. + crop_factor (float): Portion of the image to crop from the top. + + Returns: + list: A new list of bounds with the top cropped. + """ + degrees_chopped = 180 * crop_factor + new_top_start = -90 + degrees_chopped + fov / 2 + for i, el in enumerate(bound_arr): + if el < new_top_start - fov / 2: + bound_arr[i] = None + elif el < new_top_start: + diff = new_top_start - el + bound_arr[i] = new_top_start + for j in range(i + 1, len(bound_arr)): + bound_arr[j] += diff / (2 ** (j - i)) + break + + return bound_arr + + +def _crop_bound_arr_vertical( + bound_arr: list, fov: int, crop_factor: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0) +) -> list: + """Returns a list of vertical bounds adjusted for cropping. + + Args: + bound_arr (list): Original list of vertical bounds in ascending order. + fov (int): Field of view of the camera. + crop_factor (Tuple[float, float, float, float]): Crop arr (top, bottom, left, right). + + Returns: + list: Cropped bound arr + """ + if crop_factor[1] > 0: + bound_arr = _crop_bottom(bound_arr, fov, crop_factor[1]) + if crop_factor[0] > 0: + bound_arr = _crop_top(bound_arr, fov, crop_factor[0]) + return bound_arr + def generate_planar_projections_from_equirectangular( image_dir: Path, planar_image_size: Tuple[int, int], samples_per_im: int, + crop_factor: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0), ) -> Path: """Generate planar projections from an equirectangular image. @@ -44,30 +121,53 @@ def generate_planar_projections_from_equirectangular( image_dir: The directory containing the equirectangular image. planar_image_size: The size of the planar projections [width, height]. samples_per_im: The number of samples to take per image. + crop_factor: The portion of the image to crop from the (top, bottom, left, and right). + Values should be in [0, 1]. returns: The path to the planar projections directory. """ + for i in crop_factor: + if i < 0 or i > 1: + CONSOLE.print("[bold red] Invalid crop factor. All values must be in [0,1].") + sys.exit(1) + device = torch.device("cuda") fov = 120 yaw_pitch_pairs = [] + left_bound, right_bound = -180, 180 + if crop_factor[3] > 0: + left_bound = -180 + 360 * crop_factor[3] + if crop_factor[2] > 0: + right_bound = 180 - 360 * crop_factor[2] + if samples_per_im == 8: fov = 120 - for i in np.arange(-180, 180, 90): - yaw_pitch_pairs.append((i, 0)) - for i in np.arange(-180, 180, 180): - yaw_pitch_pairs.append((i, 45)) - for i in np.arange(-180, 180, 180): - yaw_pitch_pairs.append((i, -45)) + bound_arr = [-45, 0, 45] + bound_arr = _crop_bound_arr_vertical(bound_arr, fov, crop_factor) + if bound_arr[1] is not None: + for i in np.arange(left_bound, right_bound, 90): + yaw_pitch_pairs.append((i, bound_arr[1])) + if bound_arr[2] is not None: + for i in np.arange(left_bound, right_bound, 180): + yaw_pitch_pairs.append((i, bound_arr[2])) + if bound_arr[0] is not None: + for i in np.arange(left_bound, right_bound, 180): + yaw_pitch_pairs.append((i, bound_arr[0])) elif samples_per_im == 14: fov = 110 - for i in np.arange(-180, 180, 60): - yaw_pitch_pairs.append((i, 0)) - for i in np.arange(-180, 180, 90): - yaw_pitch_pairs.append((i, 45)) - for i in np.arange(-180, 180, 90): - yaw_pitch_pairs.append((i, -45)) + bound_arr = [-45, 0, 45] + bound_arr = _crop_bound_arr_vertical(bound_arr, fov, crop_factor) + if bound_arr[1] is not None: + for i in np.arange(left_bound, right_bound, 60): + yaw_pitch_pairs.append((i, bound_arr[1])) + if bound_arr[2] is not None: + for i in np.arange(left_bound, right_bound, 90): + yaw_pitch_pairs.append((i, bound_arr[2])) + if bound_arr[0] is not None: + for i in np.arange(left_bound, right_bound, 90): + yaw_pitch_pairs.append((i, bound_arr[0])) equi2pers = Equi2Pers(height=planar_image_size[1], width=planar_image_size[0], fov_x=fov, mode="bilinear") frame_dir = image_dir @@ -82,7 +182,6 @@ def generate_planar_projections_from_equirectangular( TimeRemainingColumn(elapsed_when_finished=True, compact=True), ) - curr_im = 0 with progress: for i in progress.track(os.listdir(frame_dir), description="", total=num_ims): if i.lower().endswith((".jpg", ".png", ".jpeg")): @@ -97,7 +196,6 @@ def generate_planar_projections_from_equirectangular( pers_image = (pers_image.permute(1, 2, 0)).type(torch.uint8).to("cpu").numpy() cv2.imwrite(f"{output_dir}/{i[:-4]}_{count}.jpg", pers_image) count += 1 - curr_im += 1 return output_dir diff --git a/nerfstudio/process_data/process_data_utils.py b/nerfstudio/process_data/process_data_utils.py index 6b94bbaa9..14ae33662 100644 --- a/nerfstudio/process_data/process_data_utils.py +++ b/nerfstudio/process_data/process_data_utils.py @@ -100,7 +100,11 @@ def get_num_frames_in_video(video: Path) -> int: def convert_video_to_images( - video_path: Path, image_dir: Path, num_frames_target: int, verbose: bool = False + video_path: Path, + image_dir: Path, + num_frames_target: int, + crop_factor: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0), + verbose: bool = False, ) -> Tuple[List[str], int]: """Converts a video into a sequence of images. @@ -108,11 +112,17 @@ def convert_video_to_images( video_path: Path to the video. output_dir: Path to the output directory. num_frames_target: Number of frames to extract. + crop_factor: Portion of the image to crop. Should be in [0,1] (top, bottom, left, right) verbose: If True, logs the output of the command. Returns: A tuple containing summary of the conversion and the number of extracted frames. """ + for i in crop_factor: + if i < 0 or i > 1: + CONSOLE.print("[bold red]Error: Invalid crop factor. All crops must be in [0,1].") + sys.exit(1) + if video_path.is_dir(): CONSOLE.print(f"[bold red]Error: Video path is a directory, not a path: {video_path}") sys.exit(1) @@ -135,16 +145,23 @@ def convert_video_to_images( out_filename = image_dir / "frame_%05d.png" ffmpeg_cmd = f'ffmpeg -i "{video_path}"' - spacing = num_frames // num_frames_target + crop_cmd = "" + if crop_factor != (0.0, 0.0, 0.0, 0.0): + height = 1 - crop_factor[0] - crop_factor[1] + width = 1 - crop_factor[2] - crop_factor[3] + start_x = crop_factor[2] + start_y = crop_factor[0] + crop_cmd = f',"crop=w=iw*{width}:h=ih*{height}:x=iw*{start_x}:y=ih*{start_y}"' + + spacing = num_frames // num_frames_target if spacing > 1: - ffmpeg_cmd += f" -vf thumbnail={spacing},setpts=N/TB -r 1" + ffmpeg_cmd += f" -vf thumbnail={spacing},setpts=N/TB{crop_cmd} -r 1" else: CONSOLE.print("[bold red]Can't satisfy requested number of frames. Extracting all frames.") ffmpeg_cmd += " -pix_fmt bgr8" ffmpeg_cmd += f" {out_filename}" - run_command(ffmpeg_cmd, verbose=verbose) num_final_frames = len(list(image_dir.glob("*.png"))) @@ -399,25 +416,23 @@ def generate_circle_mask(height: int, width: int, percent_radius) -> Optional[np return mask -def generate_crop_mask( - height: int, width: int, percent_crop: Tuple[float, float, float, float] -) -> Optional[np.ndarray]: +def generate_crop_mask(height: int, width: int, crop_factor: Tuple[float, float, float, float]) -> Optional[np.ndarray]: """generate a crop mask of the given size. Args: height: The height of the mask. width: The width of the mask. - percent_crop: The percent of the image to crop in each direction [top, bottom, left, right]. + crop_factor: The percent of the image to crop in each direction [top, bottom, left, right]. Returns: The mask or None if no cropping is performed. """ - if np.all(np.array(percent_crop) == 0.0): + if np.all(np.array(crop_factor) == 0.0): return None - if np.any(np.array(percent_crop) < 0.0) or np.any(np.array(percent_crop) > 1.0): + if np.any(np.array(crop_factor) < 0.0) or np.any(np.array(crop_factor) > 1.0): CONSOLE.log("[bold red]Invalid crop percentage, must be between 0 and 1.") sys.exit(1) - top, bottom, left, right = percent_crop + top, bottom, left, right = crop_factor mask = np.zeros((height, width), dtype=np.uint8) top = int(top * height) bottom = int(bottom * height) @@ -428,20 +443,20 @@ def generate_crop_mask( def generate_mask( - height: int, width: int, percent_crop: Tuple[float, float, float, float], percent_radius: float + height: int, width: int, crop_factor: Tuple[float, float, float, float], percent_radius: float ) -> Optional[np.ndarray]: """generate a mask of the given size. Args: height: The height of the mask. width: The width of the mask. - percent_crop: The percent of the image to crop in each direction [top, bottom, left, right]. + crop_factor: The percent of the image to crop in each direction [top, bottom, left, right]. percent_radius: The radius of the circle as a percentage of the image diagonal size. Returns: The mask or None if no mask is needed. """ - crop_mask = generate_crop_mask(height, width, percent_crop) + crop_mask = generate_crop_mask(height, width, crop_factor) circle_mask = generate_circle_mask(height, width, percent_radius) if crop_mask is None: return circle_mask @@ -453,7 +468,7 @@ def generate_mask( def save_mask( image_dir: Path, num_downscales: int, - percent_crop: Tuple[float, float, float, float] = (0, 0, 0, 0), + crop_factor: Tuple[float, float, float, float] = (0, 0, 0, 0), percent_radius: float = 1.0, ) -> Optional[Path]: """Save a mask for each image in the image directory. @@ -461,7 +476,7 @@ def save_mask( Args: image_dir: The directory containing the images. num_downscales: The number of downscaling levels. - percent_crop: The percent of the image to crop in each direction [top, bottom, left, right]. + crop_factor: The percent of the image to crop in each direction [top, bottom, left, right]. percent_radius: The radius of the circle as a percentage of the image diagonal size. Returns: @@ -470,7 +485,7 @@ def save_mask( image_path = next(image_dir.glob("frame_*")) image = cv2.imread(str(image_path)) height, width = image.shape[:2] - mask = generate_mask(height, width, percent_crop, percent_radius) + mask = generate_mask(height, width, crop_factor, percent_radius) if mask is None: return None mask *= 255 diff --git a/nerfstudio/viewer/app/package.json b/nerfstudio/viewer/app/package.json index 171ec8dd1..3525ebd86 100644 --- a/nerfstudio/viewer/app/package.json +++ b/nerfstudio/viewer/app/package.json @@ -1,7 +1,7 @@ { "name": "viewer", "homepage": ".", - "version": "23-02-3-0", + "version": "23-03-9-0", "private": true, "dependencies": { "@emotion/react": "^11.10.4", diff --git a/nerfstudio/viewer/app/src/index.scss b/nerfstudio/viewer/app/src/index.scss index 3a331e37a..a3af6c244 100644 --- a/nerfstudio/viewer/app/src/index.scss +++ b/nerfstudio/viewer/app/src/index.scss @@ -132,8 +132,8 @@ $sidebar-width: 100% !default; padding: 10px; } -.StatusPanel .StatusPanel-button-group { - margin: 10px; +.StatusPanel .StatusPanel-button { + margin-bottom: 4px; } .StatusPanel .StatusPanel-hide-scene-button { @@ -348,7 +348,7 @@ $sidebar-width: 100% !default; // Viewer Window .canvas-container-main, -.WebRTCVideo, +.RenderWindow, .ViewerWindow-render-crop-container { position: absolute; top: 0px; @@ -358,7 +358,7 @@ $sidebar-width: 100% !default; border: $custom-border; } -.WebRTCVideo { +.RenderWindow { z-index: 1; } @@ -373,12 +373,6 @@ $sidebar-width: 100% !default; z-index: 3; } -.WebRTCVideo-video { - width: 100%; - height: 100%; - object-fit: fill; -} - .ViewerWindow-render-crop { display: flex; box-shadow: rgb(0 0 0 / 70%) 0px 0px 0px 1600px; diff --git a/nerfstudio/viewer/app/src/modules/RenderWindow/RenderWindow.jsx b/nerfstudio/viewer/app/src/modules/RenderWindow/RenderWindow.jsx new file mode 100644 index 000000000..e2e1c53e7 --- /dev/null +++ b/nerfstudio/viewer/app/src/modules/RenderWindow/RenderWindow.jsx @@ -0,0 +1,12 @@ +import * as React from 'react'; +import { useSelector } from 'react-redux'; + +export default function RenderWindow() { + const render_img = useSelector((state) => state.render_img); + + return ( +
+ Render window +
+ ); +} diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx index 80a1c38ab..b80446664 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx @@ -1,7 +1,6 @@ import * as React from 'react'; import Button from '@mui/material/Button'; -import { ButtonGroup } from '@mui/material'; import { useDispatch, useSelector } from 'react-redux'; import PauseIcon from '@mui/icons-material/Pause'; import PlayArrowIcon from '@mui/icons-material/PlayArrow'; @@ -25,9 +24,6 @@ export default function StatusPanel(props: StatusPanelProps) { const isWebsocketConnected = useSelector( (state) => state.websocketState.isConnected, ); - const isWebrtcConnected = useSelector( - (state) => state.webrtcState.isConnected, - ); const eval_res = useSelector((state) => state.renderingState.eval_res); const vis_train_ratio = useSelector( (state) => state.renderingState.vis_train_ratio, @@ -78,13 +74,9 @@ export default function StatusPanel(props: StatusPanelProps) { const training_icon = isTraining ? : ; const websocket_connected_text = isWebsocketConnected - ? 'Server Connected' - : 'Server Disconnected'; - const webrtc_connected_text = isWebrtcConnected - ? 'Render Connected' - : 'Render Disconnected'; + ? 'Renderer Connected' + : 'Renderer Disconnected'; const websocket_connected_color = isWebsocketConnected ? 'success' : 'error'; - const webrtc_connected_color = isWebrtcConnected ? 'success' : 'error'; return (
@@ -143,26 +135,13 @@ export default function StatusPanel(props: StatusPanelProps) { Time Allocation: {vis_train_ratio}
- - - - + {websocket_connected_text} + ); } diff --git a/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx b/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx index 5218b82b8..ea6196a14 100644 --- a/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx +++ b/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx @@ -11,8 +11,8 @@ import PublicSharpIcon from '@mui/icons-material/PublicSharp'; import SyncOutlinedIcon from '@mui/icons-material/SyncOutlined'; import ThreeDRotationIcon from '@mui/icons-material/ThreeDRotation'; import VideoCameraBackIcon from '@mui/icons-material/VideoCameraBackOutlined'; -import WebRtcWindow from '../WebRtcWindow/WebRtcWindow'; import { WebSocketContext } from '../WebSocket/WebSocket'; +import RenderWindow from '../RenderWindow/RenderWindow'; const msgpack = require('msgpack-lite'); @@ -100,7 +100,7 @@ function TransformIcons(props) { ); } -// manages a camera and the web rtc stream... +// manages a camera export default function ViewerWindow(props) { const sceneTree = props.sceneTree; const scene = sceneTree.object; @@ -274,8 +274,7 @@ export default function ViewerWindow(props) { return ( <> - {/* the webrtc viewer needs to know the camera pose */} - +
diff --git a/nerfstudio/viewer/app/src/modules/WebRtcWindow/WebRtcWindow.jsx b/nerfstudio/viewer/app/src/modules/WebRtcWindow/WebRtcWindow.jsx deleted file mode 100644 index 80d69523d..000000000 --- a/nerfstudio/viewer/app/src/modules/WebRtcWindow/WebRtcWindow.jsx +++ /dev/null @@ -1,186 +0,0 @@ -import React, { createContext, useContext, useEffect, useRef } from 'react'; - -import { useDispatch } from 'react-redux'; -import { WebSocketContext } from '../WebSocket/WebSocket'; - -const WebRtcContext = createContext(null); -const msgpack = require('msgpack-lite'); - -export { WebRtcContext }; - -export default function WebRtcWindow() { - const websocket = useContext(WebSocketContext).socket; - const pcRef = useRef(null); - const localVideoRef = useRef(null); - - const dispatch = useDispatch(); - - const getRTCPeerConnection = () => { - const pc = new RTCPeerConnection({ - iceServers: [ - { urls: 'stun:stun.l.google.com:19302' }, - { - urls: 'stun:openrelay.metered.ca:80', - }, - { - urls: 'turn:openrelay.metered.ca:80', - username: 'openrelayproject', - credential: 'openrelayproject', - }, - { - urls: 'turn:openrelay.metered.ca:443', - username: 'openrelayproject', - credential: 'openrelayproject', - }, - { - urls: 'turn:openrelay.metered.ca:443?transport=tcp', - username: 'openrelayproject', - credential: 'openrelayproject', - }, - ], - }); - // connect video - pc.addEventListener('track', (evt) => { - if (evt.track.kind === 'video') { - [localVideoRef.current.srcObject] = evt.streams; // uses array destructuring - } - }); - pc.addTransceiver('video', { direction: 'recvonly' }); - - // for updating the status of the peer connection - pc.oniceconnectionstatechange = () => { - // https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/connectionState - console.log(`[webrtc] connectionState: ${pc.connectionState}`); - if ( - pc.connectionState === 'connecting' || - pc.connectionState === 'connected' - ) { - console.log('[webrtc] connected'); - dispatch({ - type: 'write', - path: 'webrtcState/isConnected', - data: true, - }); - } else { - dispatch({ - type: 'write', - path: 'webrtcState/isConnected', - data: false, - }); - } - }; - - pc.onclose = () => { - dispatch({ - type: 'write', - path: 'webrtcState/isConnected', - data: false, - }); - }; - - return pc; - }; - - const sendOffer = () => { - pcRef.current - .createOffer() - .then((offer) => { - console.log('[webrtc] created offer'); - console.log(offer); - return pcRef.current.setLocalDescription(offer); - }) - .then(() => { - // wait for ICE gathering to complete - console.log('[webrtc] set local description'); - return new Promise((resolve) => { - if (pcRef.current.iceGatheringState === 'complete') { - console.log('[webrtc] ICE gathering complete'); - resolve(); - } else { - const checkState = () => { - console.log( - `[webrtc] iceGatheringState: ${pcRef.current.iceGatheringState}`, - ); - if (pcRef.current.iceGatheringState === 'complete') { - pcRef.current.removeEventListener( - 'icegatheringstatechange', - checkState, - ); - resolve(); - } - }; - console.log( - '[webrtc] adding listener for `icegatheringstatechange`', - ); - pcRef.current.addEventListener( - 'icegatheringstatechange', - checkState, - ); - } - }); - }) - .then(() => { - // send the offer - console.log('[webrtc] sending offer'); - const offer = pcRef.current.localDescription; - const cmd = 'write'; - const path = 'webrtc/offer'; - const data = { - type: cmd, - path, - data: { - sdp: offer.sdp, - type: offer.type, - }, - }; - const message = msgpack.encode(data); - websocket.send(message); - }); - }; - - useEffect(() => { - websocket.addEventListener('message', (originalCmd) => { - // set the remote description when the offer is received - const cmd = msgpack.decode(new Uint8Array(originalCmd.data)); - if (cmd.path === '/webrtc/answer') { - console.log('[webrtc] received answer'); - const answer = cmd.data; - console.log(answer); - if (answer !== null) { - pcRef.current.setRemoteDescription(answer); - } - } - }); - - websocket.addEventListener('open', () => { - // connect webrtc when the websocket is connected - pcRef.current = getRTCPeerConnection(); - console.log('[webrtc] starting process'); - sendOffer(); - }); - - // kill the webrtc connection on dismount - return () => { - if (pcRef.current !== null) { - dispatch({ - type: 'write', - path: 'webrtcState/isConnected', - data: false, - }); - pcRef.current.close(); - } - }; - }, [websocket]); // dependency to call this whenever the websocket changes - - return ( -
-
- ); -} diff --git a/nerfstudio/viewer/app/src/reducer.js b/nerfstudio/viewer/app/src/reducer.js index d8335c80d..5aefe9c8a 100644 --- a/nerfstudio/viewer/app/src/reducer.js +++ b/nerfstudio/viewer/app/src/reducer.js @@ -6,13 +6,13 @@ const initialState = { isConnected: false, websocket_url: 'ws://localhost:7007', }, - // the webrtc connection state - webrtcState: { - isConnected: false, - }, + // for sending actual commands to the client camera_path_payload: null, populate_paths_payload: false, + + render_img: null, // The rendered images + // the rendering state renderingState: { // cameras @@ -65,15 +65,9 @@ const initialState = { sceneBox: null, cameras: null, }, - // ------------------------------------------------------------------------ - // which of the state keys to synchronize with the bridge server - sync_keys: ['renderingState', 'sceneState'], }; function setData(newState, state, path, data) { - if (path === 'colormap_options') { - newState.colormap_choice = 'default'; // eslint-disable-line no-param-reassign - } if (path.length === 1) { newState[path[0]] = data; // eslint-disable-line no-param-reassign } else { diff --git a/nerfstudio/viewer/server/server.py b/nerfstudio/viewer/server/server.py index 6c2c45476..dd2b62235 100644 --- a/nerfstudio/viewer/server/server.py +++ b/nerfstudio/viewer/server/server.py @@ -64,14 +64,10 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler): # pylint: disable=a if type_ == "write": # writes the data coming from the websocket find_node(self.bridge.state_tree, path).data = m["data"] - if m["path"] != "renderingState/camera" and m["path"] != "webrtc/offer": - # dispatch to the websockets if not a camera update! - # TODO(ethan): handle the camera properly! - # TODO: but don't update the current websocket - command = {"type": "write", "path": m["path"], "data": m["data"]} - packed_data = umsgpack.packb(command) - frames = ["write".encode("utf-8"), m["path"].encode("utf-8"), packed_data] - self.bridge.forward_to_websockets(frames, websocket_to_skip=self) + command = {"type": "write", "path": m["path"], "data": m["data"]} + packed_data = umsgpack.packb(command) + frames = ["write".encode("utf-8"), m["path"].encode("utf-8"), packed_data] + self.bridge.forward_to_websockets(frames, websocket_to_skip=self) elif type_ == "read": # reads and returns the data data = find_node(self.bridge.state_tree, path).data diff --git a/nerfstudio/viewer/server/utils.py b/nerfstudio/viewer/server/utils.py index a21b02d82..b246790ef 100644 --- a/nerfstudio/viewer/server/utils.py +++ b/nerfstudio/viewer/server/utils.py @@ -21,8 +21,6 @@ from typing import Any, Callable, Dict, List, Optional, Tuple import numpy as np import torch import zmq -from aiortc import RTCPeerConnection -from aiortc.rtcrtpsender import RTCRtpSender def get_chunks( @@ -127,17 +125,3 @@ def find_available_port(func: Callable, default_port: int, max_attempts: int = 1 raise ( Exception(f"Could not find an available port in the range: [{default_port:d}, {max_attempts + default_port:d})") ) - - -def force_codec(pc: RTCPeerConnection, sender: RTCRtpSender, forced_codec: str) -> None: - """Sets the codec preferences on a connection between sender and receiver - - Args: - pc: peer connection point - sender: sender that will send to connection point - forced_codec: codec to set - """ - kind = forced_codec.split("/")[0] - codecs = RTCRtpSender.getCapabilities(kind).codecs - transceiver = next(t for t in pc.getTransceivers() if t.sender == sender) - transceiver.setCodecPreferences([codec for codec in codecs if codec.mimeType == forced_codec]) diff --git a/nerfstudio/viewer/server/video_stream.py b/nerfstudio/viewer/server/video_stream.py deleted file mode 100644 index c50168055..000000000 --- a/nerfstudio/viewer/server/video_stream.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2022 The Nerfstudio Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Video Stream objects for WebRTC""" - -import numpy as np -from aiortc import VideoStreamTrack -from av import VideoFrame - - -class SingleFrameStreamTrack(VideoStreamTrack): - """Single Frame stream class: pushes single frames to a stream""" - - def __init__(self): - super().__init__() - self.background_frame = np.ones((480, 640, 3), dtype="uint8") * 100 # gray background - self.frame = None - self.put_frame(self.background_frame) - - def put_frame(self, frame: np.ndarray) -> None: - """Sets the current viewing frame - - Args: - frame: image to be viewed - """ - self.frame = VideoFrame.from_ndarray(frame) - - async def recv(self): - """Async method to grab and wait on frame""" - pts, time_base = await self.next_timestamp() - - frame = self.frame - frame.pts = pts - frame.time_base = time_base - return frame diff --git a/nerfstudio/viewer/server/viewer_utils.py b/nerfstudio/viewer/server/viewer_utils.py index 35b63e9f6..89d66766f 100644 --- a/nerfstudio/viewer/server/viewer_utils.py +++ b/nerfstudio/viewer/server/viewer_utils.py @@ -18,10 +18,9 @@ """ from __future__ import annotations -import asyncio +import base64 import enum import os -import re import sys import threading import time @@ -29,14 +28,9 @@ import warnings from pathlib import Path from typing import Any, Dict, Optional, Tuple +import cv2 import numpy as np import torch -from aiortc import ( - RTCConfiguration, - RTCIceServer, - RTCPeerConnection, - RTCSessionDescription, -) from cryptography.utils import CryptographyDeprecationWarning from rich.console import Console @@ -52,11 +46,7 @@ from nerfstudio.utils.decorators import check_main_thread, decorate_all from nerfstudio.utils.io import load_from_json, write_to_json from nerfstudio.utils.writer import GLOBAL_BUFFER, EventName, TimeWriter from nerfstudio.viewer.server.subprocess import run_viewer_bridge_server_as_subprocess -from nerfstudio.viewer.server.utils import ( - force_codec, - get_intrinsics_matrix_and_camera_to_world_h, -) -from nerfstudio.viewer.server.video_stream import SingleFrameStreamTrack +from nerfstudio.viewer.server.utils import get_intrinsics_matrix_and_camera_to_world_h from nerfstudio.viewer.server.visualizer import Viewer warnings.filterwarnings("ignore", category=CryptographyDeprecationWarning) @@ -284,11 +274,9 @@ class ViewerState: CONSOLE.rule(characters="=") CONSOLE.line() self.vis = Viewer(zmq_port=zmq_port, ip_address=self.config.ip_address) - self.vis_webrtc_thread = Viewer(zmq_port=zmq_port, ip_address=self.config.ip_address) else: assert self.config.zmq_port is not None self.vis = Viewer(zmq_port=self.config.zmq_port, ip_address=self.config.ip_address) - self.vis_webrtc_thread = Viewer(zmq_port=self.config.zmq_port, ip_address=self.config.ip_address) # viewer specific variables self.prev_camera_matrix = None @@ -315,12 +303,6 @@ class ViewerState: self.output_list = None - # webrtc - self.pcs = set() - self.video_tracks = set() - self.webrtc_thread = None - self.kill_webrtc_signal = False - def _pick_drawn_image_idxs(self, total_num: int) -> list[int]: """Determine indicies of images to display in viewer. @@ -351,10 +333,8 @@ class ViewerState: # set the data base dir self.vis["renderingState/data_base_dir"].write(str(self.datapath)) - # get the timestamp of the train run to set default export path name - timestamp_reg = re.compile("[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{6}") - timestamp_match = timestamp_reg.findall(str(self.log_filename.parents[0])) - self.vis["renderingState/export_path"].write(timestamp_match[-1]) + # set default export path name + self.vis["renderingState/export_path"].write(self.log_filename.parent.stem) # clear the current scene self.vis["sceneState/sceneBox"].delete() @@ -420,26 +400,6 @@ class ViewerState: self.vis["renderingState/all_camera_paths"].write(all_path_dict) self.vis["populate_paths_payload"].delete() - def _check_webrtc_offer(self): - """Check if there is a webrtc offer to respond to.""" - data = self.vis["webrtc/offer"].read() - if data: - if self.webrtc_thread and self.webrtc_thread.is_alive(): - # kill the previous thread if the webpage refreshes - self.kill_webrtc_signal = True - return - - def loop_in_thread(loop): - asyncio.set_event_loop(loop) - loop.run_until_complete(self.send_webrtc_answer(data)) - - loop = asyncio.get_event_loop() - self.webrtc_thread = threading.Thread(target=loop_in_thread, args=(loop,)) - self.webrtc_thread.daemon = True - self.webrtc_thread.start() - # remove the offer from the state tree - self.vis["webrtc/offer"].delete() - def _update_render_aabb(self, graph): """ update the render aabb box for the viewer: @@ -518,7 +478,6 @@ class ViewerState: self._check_camera_path_payload(trainer, step) self._check_populate_paths_payload(trainer, step) - self._check_webrtc_offer() camera_object = self._get_camera_object() if camera_object is None: @@ -564,7 +523,6 @@ class ViewerState: is_training = self.vis["renderingState/isTraining"].read() self._check_populate_paths_payload(trainer, step) self._check_camera_path_payload(trainer, step) - self._check_webrtc_offer() run_loop = not is_training local_step += 1 @@ -684,66 +642,6 @@ class ViewerState: raise NotImplementedError - async def send_webrtc_answer(self, data): - """Setup the webrtc connection.""" - - # returns the description to for WebRTC to the specific websocket connection - offer = RTCSessionDescription(data["sdp"], data["type"]) - - if self.config.local: - pc = RTCPeerConnection() - else: - if self.config.skip_openrelay: - ice_servers = [ - RTCIceServer(urls="stun:stun.l.google.com:19302"), - ] - else: - ice_servers = [ - RTCIceServer(urls="stun:stun.l.google.com:19302"), - RTCIceServer(urls="stun:openrelay.metered.ca:80"), - RTCIceServer( - urls="turn:openrelay.metered.ca:80", username="openrelayproject", credential="openrelayproject" - ), - RTCIceServer( - urls="turn:openrelay.metered.ca:443", username="openrelayproject", credential="openrelayproject" - ), - RTCIceServer( - urls="turn:openrelay.metered.ca:443?transport=tcp", - username="openrelayproject", - credential="openrelayproject", - ), - ] - - pc = RTCPeerConnection(configuration=RTCConfiguration(iceServers=ice_servers)) - self.pcs.add(pc) - - video = SingleFrameStreamTrack() - self.video_tracks.add(video) - video_sender = pc.addTrack(video) - print(f"viewer using video/{self.config.codec}") - force_codec(pc, video_sender, f"video/{self.config.codec}") - - await pc.setRemoteDescription(offer) - answer = await pc.createAnswer() - await pc.setLocalDescription(answer) - - self.vis_webrtc_thread["webrtc/answer"].write( - {"sdp": pc.localDescription.sdp, "type": pc.localDescription.type} - ) - self.vis_webrtc_thread["webrtc/answer"].delete() - - # continually exchange media - while True: - await asyncio.sleep(1) - if self.kill_webrtc_signal: - self.kill_webrtc_signal = False - return - - def set_image(self, image): - """Write the image over webrtc.""" - for video_track in self.video_tracks: - video_track.put_frame(image) - def _send_output_to_viewer(self, outputs: Dict[str, Any], colors: torch.Tensor = None): """Chooses the correct output and sends it to the viewer @@ -768,16 +666,22 @@ class ViewerState: if self.output_type_changed or self.prev_colormap_type is None: self.prev_colormap_type = ColormapTypes.DEFAULT colormap_options = [] + self.vis["renderingState/colormap_options"].write(list(ColormapTypes)) if outputs[reformatted_output].shape[-1] == 3: colormap_options = [ColormapTypes.DEFAULT] if outputs[reformatted_output].shape[-1] == 1 and outputs[reformatted_output].dtype == torch.float: - colormap_options = list(ColormapTypes) + self.prev_colormap_type = ColormapTypes.TURBO + colormap_options = list(ColormapTypes)[1:] self.output_type_changed = False self.vis["renderingState/colormap_choice"].write(self.prev_colormap_type) self.vis["renderingState/colormap_options"].write(colormap_options) selected_output = (self._apply_colormap(outputs, colors) * 255).type(torch.uint8) - image = selected_output.cpu().numpy() - self.set_image(image) + + image = selected_output[..., [2, 1, 0]].cpu().numpy() + + data = cv2.imencode(".jpg", image, [cv2.IMWRITE_JPEG_QUALITY, 75])[1].tobytes() + data = str("data:image/jpeg;base64," + base64.b64encode(data).decode("ascii")) + self.vis["render_img"].write(data) def _update_viewer_stats(self, render_time: float, num_rays: int, image_height: int, image_width: int) -> None: """Function that calculates and populates all the rendering statistics accordingly @@ -862,11 +766,6 @@ class ViewerState: if image_width > self.max_resolution: image_width = self.max_resolution image_height = int(image_width / aspect_ratio) - if self.config.codec != "VP8": - # force even values to allow hardware encoder usage - quantize = 2 - image_width = int(image_width / quantize) * quantize - image_height = int(image_height / quantize) * quantize return image_height, image_width def _process_invalid_output(self, output_type: str) -> str: @@ -896,7 +795,7 @@ class ViewerState: # pylint: disable=too-many-statements """ Draw an image using the current camera pose from the viewer. - The image is sent of a TCP connection and then uses WebRTC to send it to the viewer. + The image is sent over a TCP connection. Args: graph: current checkpoint of model diff --git a/pyproject.toml b/pyproject.toml index 670e22c31..8ac4f04f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,11 +14,9 @@ classifiers = [ "Programming Language :: Python", ] dependencies = [ - "aiohttp>=3.8.1", - "aiortc>=1.3.2", "appdirs>=1.4", "av>=9.2.0", - "cryptography>=38,<39", + "cryptography>=38", "tyro>=0.3.31", "gdown>=4.6.0", "ninja>=1.10", diff --git a/scripts/exporter.py b/scripts/exporter.py index a555de98c..cd18aa354 100644 --- a/scripts/exporter.py +++ b/scripts/exporter.py @@ -2,6 +2,8 @@ Script for exporting NeRF into other formats. """ +# pylint: disable=no-member + from __future__ import annotations import json @@ -94,7 +96,11 @@ class ExportPointCloud(Exporter): CONSOLE.print(f"[bold green]:white_check_mark: Generated {pcd}") CONSOLE.print("Saving Point Cloud...") - o3d.io.write_point_cloud(str(self.output_dir / "point_cloud.ply"), pcd) + tpcd = o3d.t.geometry.PointCloud.from_legacy(pcd) + # The legacy PLY writer converts colors to UInt8, + # let us do the same to save space. + tpcd.point.colors = (tpcd.point.colors * 255).to(o3d.core.Dtype.UInt8) + o3d.t.io.write_point_cloud(str(self.output_dir / "point_cloud.ply"), tpcd) print("\033[A\033[A") CONSOLE.print("[bold green]:white_check_mark: Saving Point Cloud") diff --git a/scripts/process_data.py b/scripts/process_data.py index 7c39afd1f..570fda59e 100644 --- a/scripts/process_data.py +++ b/scripts/process_data.py @@ -292,8 +292,8 @@ class ProcessVideo: """ percent_radius_crop: float = 1.0 """Create circle crop mask. The radius is the percent of the image diagonal.""" - percent_crop: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0) - """Percent of the image to crop. (top, bottom, left, right)""" + crop_factor: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0) + """Portion of the image to crop. All values should be in [0,1]. (top, bottom, left, right)""" use_sfm_depth: bool = False """If True, export and use depth maps induced from SfM points.""" include_depth_debug: bool = False @@ -319,11 +319,19 @@ class ProcessVideo: temp_image_dir = self.output_dir / "temp_images" temp_image_dir.mkdir(parents=True, exist_ok=True) summary_log, num_extracted_frames = process_data_utils.convert_video_to_images( - self.data, image_dir=temp_image_dir, num_frames_target=self.num_frames_target, verbose=self.verbose + self.data, + image_dir=temp_image_dir, + num_frames_target=self.num_frames_target, + crop_factor=(0.0, 0.0, 0.0, 0.0), + verbose=self.verbose, ) else: summary_log, num_extracted_frames = process_data_utils.convert_video_to_images( - self.data, image_dir=image_dir, num_frames_target=self.num_frames_target, verbose=self.verbose + self.data, + image_dir=image_dir, + num_frames_target=self.num_frames_target, + crop_factor=self.crop_factor, + verbose=self.verbose, ) # Generate planar projections if equirectangular @@ -332,7 +340,10 @@ class ProcessVideo: self.output_dir / "temp_images", self.images_per_equirect ) image_dir = equirect_utils.generate_planar_projections_from_equirectangular( - self.output_dir / "temp_images", perspective_image_size, self.images_per_equirect + self.output_dir / "temp_images", + perspective_image_size, + self.images_per_equirect, + crop_factor=self.crop_factor, ) # copy the perspective images to the image directory @@ -346,11 +357,11 @@ class ProcessVideo: # remove the temp_images folder shutil.rmtree(self.output_dir / "temp_images", ignore_errors=True) - # Create mask + # # Create mask mask_path = process_data_utils.save_mask( image_dir=image_dir, num_downscales=self.num_downscales, - percent_crop=self.percent_crop, + crop_factor=(0.0, 0.0, 0.0, 0.0), percent_radius=self.percent_radius_crop, ) if mask_path is not None: