diff --git a/nerfstudio/cameras/camera_optimizers.py b/nerfstudio/cameras/camera_optimizers.py index 7b7648884..4aa69a69b 100644 --- a/nerfstudio/cameras/camera_optimizers.py +++ b/nerfstudio/cameras/camera_optimizers.py @@ -33,6 +33,7 @@ from nerfstudio.configs.base_config import InstantiateConfig from nerfstudio.utils import poses as pose_utils from nerfstudio.engine.optimizers import OptimizerConfig from nerfstudio.engine.schedulers import SchedulerConfig +from nerfstudio.cameras.cameras import Cameras @dataclass @@ -146,6 +147,16 @@ class CameraOptimizer(nn.Module): raybundle.origins = raybundle.origins + correction_matrices[:, :3, 3] raybundle.directions = torch.bmm(correction_matrices[:, :3, :3], raybundle.directions[..., None]).squeeze() + def apply_to_camera(self, camera: Cameras) -> None: + """Apply the pose correction to the raybundle""" + if self.config.mode != "off": + assert camera.metadata is not None, "Must provide id of camera in its metadata" + assert "cam_idx" in camera.metadata, "Must provide id of camera in its metadata" + camera_idx = camera.metadata["cam_idx"] + adj = self([camera_idx]) # type: ignore + adj = torch.cat([adj, torch.Tensor([0, 0, 0, 1])[None, None].to(adj)], dim=1) + camera.camera_to_worlds = torch.bmm(camera.camera_to_worlds, adj) + def get_loss_dict(self, loss_dict: dict) -> None: """Add regularization""" if self.config.mode != "off": diff --git a/nerfstudio/configs/base_config.py b/nerfstudio/configs/base_config.py index c04c11aaa..85c205c5c 100644 --- a/nerfstudio/configs/base_config.py +++ b/nerfstudio/configs/base_config.py @@ -144,7 +144,7 @@ class ViewerConfig(PrintableConfig): """Whether to kill the training job when it has completed. Note this will stop rendering in the viewer.""" image_format: Literal["jpeg", "png"] = "jpeg" """Image format viewer should use; jpeg is lossy compression, while png is lossless.""" - jpeg_quality: int = 90 + jpeg_quality: int = 75 """Quality tradeoff to use for jpeg compression.""" make_share_url: bool = False """Viewer beta feature: print a shareable URL. `vis` must be set to viewer_beta; this flag is otherwise ignored.""" diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index 74cabbafc..2c3a8aac7 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -22,7 +22,6 @@ from collections import OrderedDict from typing import Dict import tyro -from nerfstudio.data.pixel_samplers import PairPixelSamplerConfig from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig from nerfstudio.configs.base_config import ViewerConfig @@ -49,6 +48,7 @@ from nerfstudio.data.dataparsers.sitcoms3d_dataparser import Sitcoms3DDataParser from nerfstudio.data.datasets.depth_dataset import DepthDataset from nerfstudio.data.datasets.sdf_dataset import SDFDataset from nerfstudio.data.datasets.semantic_dataset import SemanticDataset +from nerfstudio.data.pixel_samplers import PairPixelSamplerConfig from nerfstudio.engine.optimizers import AdamOptimizerConfig, RAdamOptimizerConfig from nerfstudio.engine.schedulers import ( CosineDecaySchedulerConfig, @@ -59,8 +59,10 @@ from nerfstudio.engine.trainer import TrainerConfig from nerfstudio.field_components.temporal_distortions import TemporalDistortionKind from nerfstudio.fields.sdf_field import SDFFieldConfig from nerfstudio.models.depth_nerfacto import DepthNerfactoModelConfig +from nerfstudio.models.gaussian_splatting import GaussianSplattingModelConfig from nerfstudio.models.generfacto import GenerfactoModelConfig from nerfstudio.models.instant_ngp import InstantNGPModelConfig +from nerfstudio.data.dataparsers.colmap_dataparser import ColmapDataParserConfig from nerfstudio.models.mipnerf import MipNerfModel from nerfstudio.models.nerfacto import NerfactoModelConfig from nerfstudio.models.neus import NeuSModelConfig @@ -69,6 +71,7 @@ from nerfstudio.models.semantic_nerfw import SemanticNerfWModelConfig from nerfstudio.models.tensorf import TensoRFModelConfig from nerfstudio.models.vanilla_nerf import NeRFModel, VanillaModelConfig from nerfstudio.pipelines.base_pipeline import VanillaPipelineConfig +from nerfstudio.data.datamanagers.full_images_datamanager import FullImageDatamanagerConfig from nerfstudio.pipelines.dynamic_batch import DynamicBatchPipelineConfig from nerfstudio.plugins.registry import discover_methods @@ -87,6 +90,7 @@ descriptions = { "generfacto": "Generative Text to NeRF model", "neus": "Implementation of NeuS. (slow)", "neus-facto": "Implementation of NeuS-Facto. (slow)", + "gaussian-splatting": "Gaussian Splatting model", } method_configs["nerfacto"] = TrainerConfig( @@ -594,6 +598,54 @@ method_configs["neus-facto"] = TrainerConfig( vis="viewer", ) +method_configs["gaussian-splatting"] = TrainerConfig( + method_name="gaussian-splatting", + steps_per_eval_image=100, + steps_per_eval_batch=100, + steps_per_save=2000, + steps_per_eval_all_images=1000, + max_num_iterations=30000, + mixed_precision=False, + gradient_accumulation_steps={"camera_opt": 100, "color": 2}, + pipeline=VanillaPipelineConfig( + datamanager=FullImageDatamanagerConfig( + dataparser=ColmapDataParserConfig(load_3D_points=True), + ), + model=GaussianSplattingModelConfig(), + ), + optimizers={ + "xyz": { + "optimizer": AdamOptimizerConfig(lr=1.6e-4, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig( + lr_final=1.6e-6, + max_steps=30000, + ), + }, + "color": { + "optimizer": AdamOptimizerConfig(lr=5e-4, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig( + lr_final=1e-4, + max_steps=30000, + ), + }, + "opacity": { + "optimizer": AdamOptimizerConfig(lr=0.05, eps=1e-15), + "scheduler": None, + }, + "scaling": { + "optimizer": AdamOptimizerConfig(lr=0.005, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-3, max_steps=30000), + }, + "rotation": {"optimizer": AdamOptimizerConfig(lr=0.001, eps=1e-15), "scheduler": None}, + "camera_opt": { + "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=5e-5, max_steps=30000), + }, + }, + viewer=ViewerConfig(num_rays_per_chunk=1 << 15), + vis="viewer_beta", +) + def merge_methods(methods, method_descriptions, new_methods, new_descriptions, overwrite=True): """Merge new methods and descriptions into existing methods and descriptions. diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index bf12bd604..67180bea6 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -21,12 +21,13 @@ from __future__ import annotations from abc import abstractmethod from collections import defaultdict from dataclasses import dataclass, field -from pathlib import Path from functools import cached_property +from pathlib import Path from typing import ( Any, Callable, Dict, + ForwardRef, Generic, List, Literal, @@ -35,9 +36,8 @@ from typing import ( Type, Union, cast, - ForwardRef, - get_origin, get_args, + get_origin, ) import torch @@ -47,7 +47,7 @@ from torch.utils.data.distributed import DistributedSampler from typing_extensions import TypeVar from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig -from nerfstudio.cameras.cameras import CameraType +from nerfstudio.cameras.cameras import Cameras, CameraType from nerfstudio.cameras.rays import RayBundle from nerfstudio.configs.base_config import InstantiateConfig from nerfstudio.configs.dataparser_configs import AnnotatedDataParserUnion @@ -55,9 +55,9 @@ from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs from nerfstudio.data.dataparsers.blender_dataparser import BlenderDataParserConfig from nerfstudio.data.datasets.base_dataset import InputDataset from nerfstudio.data.pixel_samplers import ( + PatchPixelSamplerConfig, PixelSampler, PixelSamplerConfig, - PatchPixelSamplerConfig, ) from nerfstudio.data.utils.dataloaders import ( CacheDataloader, @@ -67,9 +67,8 @@ from nerfstudio.data.utils.dataloaders import ( from nerfstudio.data.utils.nerfstudio_collate import nerfstudio_collate from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes from nerfstudio.model_components.ray_generators import RayGenerator -from nerfstudio.utils.misc import IterableWrapper +from nerfstudio.utils.misc import IterableWrapper, get_orig_class from nerfstudio.utils.rich_utils import CONSOLE -from nerfstudio.utils.misc import get_orig_class def variable_res_collate(batch: List[Dict]) -> Dict: @@ -131,7 +130,7 @@ class DataManager(nn.Module): To get data, use the next_train and next_eval functions. This data manager's next_train and next_eval methods will return 2 things: - 1. A Raybundle: This will contain the rays we are sampling, with latents and + 1. 'rays': This will contain the rays or camera we are sampling, with latents and conditionals attached (everything needed at inference) 2. A "batch" of auxiliary information: This will contain the mask, the ground truth pixels, etc needed to actually train, score, etc the model @@ -246,7 +245,7 @@ class DataManager(nn.Module): """Sets up the data manager for evaluation""" @abstractmethod - def next_train(self, step: int) -> Tuple[RayBundle, Dict]: + def next_train(self, step: int) -> Tuple[Union[RayBundle, Cameras], Dict]: """Returns the next batch of data from the train data manager. Args: @@ -258,25 +257,25 @@ class DataManager(nn.Module): raise NotImplementedError @abstractmethod - def next_eval(self, step: int) -> Tuple[RayBundle, Dict]: + def next_eval(self, step: int) -> Tuple[Union[RayBundle, Cameras], Dict]: """Returns the next batch of data from the eval data manager. Args: step: the step number of the eval image to retrieve Returns: - A tuple of the ray bundle for the image, and a dictionary of additional batch information + A tuple of the ray/camera for the image, and a dictionary of additional batch information such as the groundtruth image. """ raise NotImplementedError @abstractmethod - def next_eval_image(self, step: int) -> Tuple[int, RayBundle, Dict]: + def next_eval_image(self, step: int) -> Tuple[Cameras, Dict]: """Retrieve the next eval image. Args: step: the step number of the eval image to retrieve Returns: - A tuple of the step number, the ray bundle for the image, and a dictionary of + A tuple of the step number, the ray/camera for the image, and a dictionary of additional batch information such as the groundtruth image. """ raise NotImplementedError @@ -313,7 +312,7 @@ class DataManager(nn.Module): @dataclass class VanillaDataManagerConfig(DataManagerConfig): - """A basic data manager""" + """A basic data manager for a ray-based model""" _target: Type = field(default_factory=lambda: VanillaDataManager) """Target class to instantiate.""" @@ -555,11 +554,10 @@ class VanillaDataManager(DataManager, Generic[TDataset]): ray_bundle = self.eval_ray_generator(ray_indices) return ray_bundle, batch - def next_eval_image(self, step: int) -> Tuple[int, RayBundle, Dict]: - for camera_ray_bundle, batch in self.eval_dataloader: - assert camera_ray_bundle.camera_indices is not None - image_idx = int(camera_ray_bundle.camera_indices[0, 0, 0]) - return image_idx, camera_ray_bundle, batch + def next_eval_image(self, step: int) -> Tuple[Cameras, Dict]: + for camera, batch in self.eval_dataloader: + assert camera.shape[0] == 1 + return camera, batch raise ValueError("No more eval images") def get_train_rays_per_batch(self) -> int: diff --git a/nerfstudio/data/datamanagers/full_images_datamanager.py b/nerfstudio/data/datamanagers/full_images_datamanager.py new file mode 100644 index 000000000..35837f05a --- /dev/null +++ b/nerfstudio/data/datamanagers/full_images_datamanager.py @@ -0,0 +1,410 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Data manager that outputs cameras / images instead of raybundles + +Good for things like gaussian splatting which require full cameras instead of the standard ray +paradigm +""" + +from __future__ import annotations + +import random +from dataclasses import dataclass, field +from functools import cached_property +from pathlib import Path +from typing import Dict, ForwardRef, Generic, List, Literal, Optional, Tuple, Type, Union, cast, get_args, get_origin + +import cv2 +import numpy as np +import torch +from copy import deepcopy +from torch.nn import Parameter +from tqdm import tqdm + +from nerfstudio.cameras.cameras import Cameras, CameraType +from nerfstudio.configs.dataparser_configs import AnnotatedDataParserUnion +from nerfstudio.data.datamanagers.base_datamanager import DataManager, DataManagerConfig, TDataset +from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs +from nerfstudio.data.dataparsers.nerfstudio_dataparser import NerfstudioDataParserConfig +from nerfstudio.data.datasets.base_dataset import InputDataset +from nerfstudio.utils.misc import get_orig_class +from nerfstudio.utils.rich_utils import CONSOLE + + +@dataclass +class FullImageDatamanagerConfig(DataManagerConfig): + _target: Type = field(default_factory=lambda: FullImageDatamanager) + dataparser: AnnotatedDataParserUnion = NerfstudioDataParserConfig() + camera_res_scale_factor: float = 1.0 + """The scale factor for scaling spatial data such as images, mask, semantics + along with relevant information about camera intrinsics + """ + eval_num_images_to_sample_from: int = -1 + """Number of images to sample during eval iteration.""" + eval_num_times_to_repeat_images: int = -1 + """When not evaluating on all images, number of iterations before picking + new images. If -1, never pick new images.""" + eval_image_indices: Optional[Tuple[int, ...]] = (0,) + """Specifies the image indices to use during eval; if None, uses all.""" + cache_images: Literal["no-cache", "cpu", "gpu"] = "cpu" + """Whether to cache images in memory. If "numpy", caches as numpy arrays, if "torch", caches as torch tensors.""" + + +class FullImageDatamanager(DataManager, Generic[TDataset]): + """ + A datamanager that outputs full images and cameras instead of raybundles. This makes the + datamanager more lightweight since we don't have to do generate rays. Useful for full-image + training e.g. rasterization pipelines + """ + + config: FullImageDatamanagerConfig + train_dataset: TDataset + eval_dataset: TDataset + + def __init__( + self, + config: FullImageDatamanagerConfig, + device: Union[torch.device, str] = "cpu", + test_mode: Literal["test", "val", "inference"] = "val", + world_size: int = 1, + local_rank: int = 0, + **kwargs, + ): + self.config = config + self.device = device + self.world_size = world_size + self.local_rank = local_rank + self.sampler = None + self.test_mode = test_mode + self.test_split = "test" if test_mode in ["test", "inference"] else "val" + self.dataparser_config = self.config.dataparser + if self.config.data is not None: + self.config.dataparser.data = Path(self.config.data) + else: + self.config.data = self.config.dataparser.data + self.dataparser = self.dataparser_config.setup() + if test_mode == "inference": + self.dataparser.downscale_factor = 1 # Avoid opening images + self.includes_time = self.dataparser.includes_time + + self.train_dataparser_outputs: DataparserOutputs = self.dataparser.get_dataparser_outputs(split="train") + self.train_dataset = self.create_train_dataset() + self.eval_dataset = self.create_eval_dataset() + if len(self.train_dataset) > 500 and self.config.cache_images == "gpu": + CONSOLE.print("Train dataset has over 500 images, overriding cach_images to cpu", style="bold yellow") + self.config.cache_images = "cpu" + self.cached_train, self.cached_eval = self.cache_images(self.config.cache_images) + self.exclude_batch_keys_from_device = self.train_dataset.exclude_batch_keys_from_device + if self.config.masks_on_gpu is True: + self.exclude_batch_keys_from_device.remove("mask") + if self.config.images_on_gpu is True: + self.exclude_batch_keys_from_device.remove("image") + + # Some logic to make sure we sample every camera in equal amounts + self.train_unseen_cameras = [i for i in range(len(self.train_dataset))] + self.eval_unseen_cameras = [i for i in range(len(self.eval_dataset))] + assert len(self.train_unseen_cameras) > 0, "No data found in dataset" + + super().__init__() + + def cache_images(self, cache_images_option): + cached_train = [] + CONSOLE.log("Caching / undistorting train images") + for i in tqdm(range(len(self.train_dataset)), leave=False): + # cv2.undistort the images / cameras + data = self.train_dataset.get_data(i) + camera = self.train_dataset.cameras[i].reshape(()) + K = camera.get_intrinsics_matrices().numpy() + if camera.distortion_params is None: + continue + distortion_params = camera.distortion_params.numpy() + image = data["image"].numpy() + + if camera.camera_type.item() == CameraType.PERSPECTIVE.value: + distortion_params = np.array( + [ + distortion_params[0], + distortion_params[1], + distortion_params[4], + distortion_params[5], + distortion_params[2], + distortion_params[3], + 0, + 0, + ] + ) + newK, roi = cv2.getOptimalNewCameraMatrix(K, distortion_params, (image.shape[1], image.shape[0]), 0) + image = cv2.undistort(image, K, distortion_params, None, newK) # type: ignore + # crop the image and update the intrinsics accordingly + x, y, w, h = roi + image = image[y : y + h, x : x + w] + if "mask" in data: + data["mask"] = data["mask"][y : y + h, x : x + w] + if "depth_image" in data: + data["depth_image"] = data["depth_image"][y : y + h, x : x + w] + # update the width, height + self.train_dataset.cameras.width[i] = w + self.train_dataset.cameras.height[i] = h + if "mask" in data: + mask = data["mask"].numpy() + mask = mask.astype(np.uint8) * 255 + mask = cv2.undistort(mask, K, distortion_params, None, newK) # type: ignore + mask = mask[y : y + h, x : x + w] + data["mask"] = torch.from_numpy(mask).bool() + K = newK + + elif camera.camera_type.item() == CameraType.FISHEYE.value: + distortion_params = np.array( + [distortion_params[0], distortion_params[1], distortion_params[2], distortion_params[3]] + ) + newK = cv2.fisheye.estimateNewCameraMatrixForUndistortRectify( + K, distortion_params, (image.shape[1], image.shape[0]), np.eye(3), balance=0 + ) + map1, map2 = cv2.fisheye.initUndistortRectifyMap( + K, distortion_params, np.eye(3), newK, (image.shape[1], image.shape[0]), cv2.CV_32FC1 + ) + # and then remap: + image = cv2.remap(image, map1, map2, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT) + if "mask" in data: + mask = data["mask"].numpy() + mask = mask.astype(np.uint8) * 255 + mask = cv2.fisheye.undistortImage(mask, K, distortion_params, None, newK) + data["mask"] = torch.from_numpy(mask).bool() + K = newK + else: + raise NotImplementedError("Only perspective and fisheye cameras are supported") + data["image"] = torch.from_numpy(image) + + cached_train.append(data) + + self.train_dataset.cameras.fx[i] = float(K[0, 0]) + self.train_dataset.cameras.fy[i] = float(K[1, 1]) + self.train_dataset.cameras.cx[i] = float(K[0, 2]) + self.train_dataset.cameras.cy[i] = float(K[1, 2]) + + cached_eval = [] + CONSOLE.log("Caching / undistorting eval images") + for i in tqdm(range(len(self.eval_dataset)), leave=False): + # cv2.undistort the images / cameras + data = self.eval_dataset.get_data(i) + camera = self.eval_dataset.cameras[i].reshape(()) + K = camera.get_intrinsics_matrices().numpy() + if camera.distortion_params is None: + continue + distortion_params = camera.distortion_params.numpy() + image = data["image"].numpy() + + if camera.camera_type.item() == CameraType.PERSPECTIVE.value: + distortion_params = np.array( + [ + distortion_params[0], + distortion_params[1], + distortion_params[4], + distortion_params[5], + distortion_params[2], + distortion_params[3], + 0, + 0, + ] + ) + newK, roi = cv2.getOptimalNewCameraMatrix(K, distortion_params, (image.shape[1], image.shape[0]), 0) + image = cv2.undistort(image, K, distortion_params, None, newK) # type: ignore + # crop the image and update the intrinsics accordingly + x, y, w, h = roi + image = image[y : y + h, x : x + w] + # update the width, height + self.eval_dataset.cameras.width[i] = w + self.eval_dataset.cameras.height[i] = h + if "mask" in data: + mask = data["mask"].numpy() + mask = mask.astype(np.uint8) * 255 + mask = cv2.undistort(mask, K, distortion_params, None, newK) # type: ignore + mask = mask[y : y + h, x : x + w] + data["mask"] = torch.from_numpy(mask).bool() + K = newK + + elif camera.camera_type.item() == CameraType.FISHEYE.value: + distortion_params = np.array( + [distortion_params[0], distortion_params[1], distortion_params[2], distortion_params[3]] + ) + newK = cv2.fisheye.estimateNewCameraMatrixForUndistortRectify( + K, distortion_params, (image.shape[1], image.shape[0]), np.eye(3), balance=0 + ) + map1, map2 = cv2.fisheye.initUndistortRectifyMap( + K, distortion_params, np.eye(3), newK, (image.shape[1], image.shape[0]), cv2.CV_32FC1 + ) + # and then remap: + image = cv2.remap(image, map1, map2, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT) + if "mask" in data: + mask = data["mask"].numpy() + mask = mask.astype(np.uint8) * 255 + mask = cv2.fisheye.undistortImage(mask, K, distortion_params, None, newK) + data["mask"] = torch.from_numpy(mask).bool() + K = newK + else: + raise NotImplementedError("Only perspective and fisheye cameras are supported") + data["image"] = torch.from_numpy(image) + + cached_eval.append(data) + + self.eval_dataset.cameras.fx[i] = float(K[0, 0]) + self.eval_dataset.cameras.fy[i] = float(K[1, 1]) + self.eval_dataset.cameras.cx[i] = float(K[0, 2]) + self.eval_dataset.cameras.cy[i] = float(K[1, 2]) + + if cache_images_option == "gpu": + for cache in cached_train: + cache["image"] = cache["image"].to(self.device) + if "mask" in cache: + cache["mask"] = cache["mask"].to(self.device) + for cache in cached_eval: + cache["image"] = cache["image"].to(self.device) + if "mask" in cache: + cache["mask"] = cache["mask"].to(self.device) + else: + for cache in cached_train: + cache["image"] = cache["image"].pin_memory() + if "mask" in cache: + cache["mask"] = cache["mask"].pin_memory() + for cache in cached_eval: + cache["image"] = cache["image"].pin_memory() + if "mask" in cache: + cache["mask"] = cache["mask"].pin_memory() + + return cached_train, cached_eval + + def create_train_dataset(self) -> TDataset: + """Sets up the data loaders for training""" + return self.dataset_type( + dataparser_outputs=self.train_dataparser_outputs, + scale_factor=self.config.camera_res_scale_factor, + ) + + def create_eval_dataset(self) -> TDataset: + """Sets up the data loaders for evaluation""" + return self.dataset_type( + dataparser_outputs=self.dataparser.get_dataparser_outputs(split=self.test_split), + scale_factor=self.config.camera_res_scale_factor, + ) + + @cached_property + def dataset_type(self) -> Type[TDataset]: + """Returns the dataset type passed as the generic argument""" + default: Type[TDataset] = cast(TDataset, TDataset.__default__) # type: ignore + orig_class: Type[FullImageDatamanager] = get_orig_class(self, default=None) # type: ignore + if type(self) is FullImageDatamanager and orig_class is None: + return default + if orig_class is not None and get_origin(orig_class) is FullImageDatamanager: + return get_args(orig_class)[0] + + # For inherited classes, we need to find the correct type to instantiate + for base in getattr(self, "__orig_bases__", []): + if get_origin(base) is FullImageDatamanager: + for value in get_args(base): + if isinstance(value, ForwardRef): + if value.__forward_evaluated__: + value = value.__forward_value__ + elif value.__forward_module__ is None: + value.__forward_module__ = type(self).__module__ + value = getattr(value, "_evaluate")(None, None, set()) + assert isinstance(value, type) + if issubclass(value, InputDataset): + return cast(Type[TDataset], value) + return default + + def get_datapath(self) -> Path: + return self.config.dataparser.data + + def setup_train(self): + """Sets up the data loaders for training""" + + def setup_eval(self): + """Sets up the data loader for evaluation""" + + @property + def fixed_indices_eval_dataloader(self) -> List[Tuple[Cameras, Dict]]: + """ + Pretends to be the dataloader for evaluation, it returns a list of (camera, data) tuples + """ + image_indices = list(range(len(self.eval_unseen_cameras))) + data = deepcopy(self.cached_eval) + _cameras = deepcopy(self.eval_dataset.cameras).to(self.device) + cameras = [] + for i in image_indices: + data[i]["image"] = data[i]["image"].to(self.device) + cameras.append(_cameras[i : i + 1]) + assert len(self.eval_dataset.cameras.shape) == 1, "Assumes single batch dimension" + return list(zip(cameras, data)) + + def get_param_groups(self) -> Dict[str, List[Parameter]]: + """Get the param groups for the data manager. + Returns: + A list of dictionaries containing the data manager's param groups. + """ + return {} + + def get_train_rays_per_batch(self): + # TODO: fix this to be the resolution of the last image rendered + return 800 * 800 + + def next_train(self, step: int) -> Tuple[Cameras, Dict]: + """Returns the next training batch + + Returns a Camera instead of raybundle""" + image_idx = self.train_unseen_cameras.pop(random.randint(0, len(self.train_unseen_cameras) - 1)) + # Make sure to re-populate the unseen cameras list if we have exhausted it + if len(self.train_unseen_cameras) == 0: + self.train_unseen_cameras = [i for i in range(len(self.train_dataset))] + + data = deepcopy(self.cached_train[image_idx]) + data["image"] = data["image"].to(self.device) + + assert len(self.train_dataset.cameras.shape) == 1, "Assumes single batch dimension" + camera = self.train_dataset.cameras[image_idx : image_idx + 1].to(self.device) + if camera.metadata is None: + camera.metadata = {} + camera.metadata["cam_idx"] = image_idx + return camera, data + + def next_eval(self, step: int) -> Tuple[Cameras, Dict]: + """Returns the next evaluation batch + + Returns a Camera instead of raybundle""" + image_idx = self.eval_unseen_cameras.pop(random.randint(0, len(self.eval_unseen_cameras) - 1)) + # Make sure to re-populate the unseen cameras list if we have exhausted it + if len(self.eval_unseen_cameras) == 0: + self.eval_unseen_cameras = [i for i in range(len(self.eval_dataset))] + data = deepcopy(self.cached_eval[image_idx]) + data["image"] = data["image"].to(self.device) + assert len(self.eval_dataset.cameras.shape) == 1, "Assumes single batch dimension" + camera = self.eval_dataset.cameras[image_idx : image_idx + 1].to(self.device) + return camera, data + + def next_eval_image(self, step: int) -> Tuple[Cameras, Dict]: + """Returns the next evaluation batch + + Returns a Camera instead of raybundle + + TODO: Make sure this logic is consistent with the vanilladatamanager""" + image_idx = self.eval_unseen_cameras.pop(random.randint(0, len(self.eval_unseen_cameras) - 1)) + # Make sure to re-populate the unseen cameras list if we have exhausted it + if len(self.eval_unseen_cameras) == 0: + self.eval_unseen_cameras = [i for i in range(len(self.eval_dataset))] + data = deepcopy(self.cached_eval[image_idx]) + data["image"] = data["image"].to(self.device) + assert len(self.eval_dataset.cameras.shape) == 1, "Assumes single batch dimension" + camera = self.eval_dataset.cameras[image_idx : image_idx + 1].to(self.device) + return camera, data diff --git a/nerfstudio/data/datamanagers/parallel_datamanager.py b/nerfstudio/data/datamanagers/parallel_datamanager.py index 5c4d5c4a2..4e22bee15 100644 --- a/nerfstudio/data/datamanagers/parallel_datamanager.py +++ b/nerfstudio/data/datamanagers/parallel_datamanager.py @@ -38,7 +38,7 @@ import torch.multiprocessing as mp from rich.progress import track from torch.nn import Parameter -from nerfstudio.cameras.cameras import CameraType +from nerfstudio.cameras.cameras import Cameras, CameraType from nerfstudio.cameras.rays import RayBundle from nerfstudio.data.datamanagers.base_datamanager import ( DataManager, @@ -293,12 +293,11 @@ class ParallelDataManager(DataManager, Generic[TDataset]): ray_bundle = self.eval_ray_generator(ray_indices) return ray_bundle, batch - def next_eval_image(self, step: int) -> Tuple[int, RayBundle, Dict]: + def next_eval_image(self, step: int) -> Tuple[Cameras, Dict]: """Retrieve the next eval image.""" - for camera_ray_bundle, batch in self.eval_dataloader: - assert camera_ray_bundle.camera_indices is not None - image_idx = int(camera_ray_bundle.camera_indices[0, 0, 0]) - return image_idx, camera_ray_bundle, batch + for camera, batch in self.eval_dataloader: + assert camera.shape[0] == 1 + return camera, batch raise ValueError("No more eval images") def get_train_rays_per_batch(self) -> int: diff --git a/nerfstudio/data/datamanagers/random_cameras_datamanager.py b/nerfstudio/data/datamanagers/random_cameras_datamanager.py index 4269b428a..7704f2f16 100644 --- a/nerfstudio/data/datamanagers/random_cameras_datamanager.py +++ b/nerfstudio/data/datamanagers/random_cameras_datamanager.py @@ -290,11 +290,10 @@ class RandomCamerasDataManager(DataManager): # pylint: disable=abstract-method return ray_bundle, {"vertical": vertical_rotation, "central": central_rotation} - def next_eval_image(self, step: int) -> Tuple[int, RayBundle, Dict]: - for camera_ray_bundle, batch in self.eval_dataloader: - assert camera_ray_bundle.camera_indices is not None - image_idx = int(camera_ray_bundle.camera_indices[0, 0, 0]) - return image_idx, camera_ray_bundle, batch + def next_eval_image(self, step: int) -> Tuple[Cameras, Dict]: + for camera, batch in self.eval_dataloader: + assert camera.shape[0] == 1 + return camera, batch raise ValueError("No more eval images") def get_train_rays_per_batch(self) -> int: diff --git a/nerfstudio/data/dataparsers/colmap_dataparser.py b/nerfstudio/data/dataparsers/colmap_dataparser.py index 071a11e1e..02ddca293 100644 --- a/nerfstudio/data/dataparsers/colmap_dataparser.py +++ b/nerfstudio/data/dataparsers/colmap_dataparser.py @@ -70,7 +70,7 @@ class ColmapDataParserConfig(DataParserConfig): """Path to masks directory. If not set, masks are not loaded.""" depths_path: Optional[Path] = None """Path to depth maps directory. If not set, depths are not loaded.""" - colmap_path: Path = Path("sparse/0") + colmap_path: Path = Path("colmap/sparse/0") """Path to the colmap reconstruction directory relative to the data path.""" load_3D_points: bool = False """Whether to load the 3D points from the colmap reconstruction.""" diff --git a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py index 9b4c2951a..7f6f21b88 100644 --- a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py +++ b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py @@ -20,6 +20,7 @@ from pathlib import Path from typing import Literal, Optional, Type import numpy as np +import open3d as o3d import torch from PIL import Image @@ -318,6 +319,12 @@ class Nerfstudio(DataParser): applied_scale = float(meta["applied_scale"]) scale_factor *= applied_scale + # Load 3D points + metadata = {} + if "ply_file_path" in meta: + ply_file_path = data_dir / meta["ply_file_path"] + metadata.update(self._load_3D_points(ply_file_path, transform_matrix, scale_factor)) + dataparser_outputs = DataparserOutputs( image_filenames=image_filenames, cameras=cameras, @@ -328,10 +335,34 @@ class Nerfstudio(DataParser): metadata={ "depth_filenames": depth_filenames if len(depth_filenames) > 0 else None, "depth_unit_scale_factor": self.config.depth_unit_scale_factor, + **metadata, }, ) return dataparser_outputs + def _load_3D_points(self, ply_file_path: Path, transform_matrix: torch.Tensor, scale_factor: float): + pcd = o3d.io.read_point_cloud(str(ply_file_path)) + + points3D = torch.from_numpy(np.asarray(pcd.points, dtype=np.float32)) + points3D = ( + torch.cat( + ( + points3D, + torch.ones_like(points3D[..., :1]), + ), + -1, + ) + @ transform_matrix.T + ) + points3D *= scale_factor + points3D_rgb = torch.from_numpy((np.asarray(pcd.colors) * 255).astype(np.uint8)) + + out = { + "points3D_xyz": points3D, + "points3D_rgb": points3D_rgb, + } + return out + def _get_fname(self, filepath: Path, data_dir: Path, downsample_folder_prefix="images_") -> Path: """Get the filename of the image file. downsample_folder_prefix can be used to point to auxiliary image data, e.g. masks diff --git a/nerfstudio/data/utils/dataloaders.py b/nerfstudio/data/utils/dataloaders.py index e546a130b..6a64ba738 100644 --- a/nerfstudio/data/utils/dataloaders.py +++ b/nerfstudio/data/utils/dataloaders.py @@ -175,13 +175,17 @@ class EvalDataloader(DataLoader): def __next__(self) -> Tuple[RayBundle, Dict]: """Returns the next batch of data""" - def get_camera(self, image_idx: int = 0) -> Cameras: + def get_camera(self, image_idx: int = 0) -> Tuple[Cameras, Dict]: """Get camera for the given image index Args: image_idx: Camera image index """ - return self.cameras[image_idx] + camera = self.cameras[image_idx : image_idx + 1] + batch = self.input_dataset[image_idx] + batch = get_dict_to_torch(batch, device=self.device, exclude=["image"]) + assert isinstance(batch, dict) + return camera, batch def get_data_from_image_idx(self, image_idx: int) -> Tuple[RayBundle, Dict]: """Returns the data for a specific image index. @@ -226,9 +230,9 @@ class FixedIndicesEvalDataloader(EvalDataloader): def __next__(self): if self.count < len(self.image_indices): image_idx = self.image_indices[self.count] - ray_bundle, batch = self.get_data_from_image_idx(image_idx) + camera, batch = self.get_camera(image_idx) self.count += 1 - return ray_bundle, batch + return camera, batch raise StopIteration @@ -245,5 +249,5 @@ class RandIndicesEvalDataloader(EvalDataloader): def __next__(self): # choose a random image index image_idx = random.randint(0, len(self.cameras) - 1) - ray_bundle, batch = self.get_data_from_image_idx(image_idx) - return ray_bundle, batch + camera, batch = self.get_camera(image_idx) + return camera, batch diff --git a/nerfstudio/engine/optimizers.py b/nerfstudio/engine/optimizers.py index 9b8d48dba..0a8794786 100644 --- a/nerfstudio/engine/optimizers.py +++ b/nerfstudio/engine/optimizers.py @@ -135,6 +135,12 @@ class Optimizers: for _, optimizer in self.optimizers.items(): optimizer.zero_grad() + def zero_grad_some(self, param_groups: List[str]) -> None: + """Zero the gradients for the given parameter groups.""" + for param_group in param_groups: + optimizer = self.optimizers[param_group] + optimizer.zero_grad() + def optimizer_scaler_step_all(self, grad_scaler: GradScaler) -> None: """Take an optimizer step using a grad scaler. @@ -149,6 +155,21 @@ class Optimizers: if any(any(p.grad is not None for p in g["params"]) for g in optimizer.param_groups): grad_scaler.step(optimizer) + def optimizer_scaler_step_some(self, grad_scaler: GradScaler, param_groups: List[str]) -> None: + """Take an optimizer step using a grad scaler ONLY on the specified param groups. + + Args: + grad_scaler: GradScaler to use + """ + for param_group in param_groups: + optimizer = self.optimizers[param_group] + max_norm = self.config[param_group]["optimizer"].max_norm + if max_norm is not None: + grad_scaler.unscale_(optimizer) + torch.nn.utils.clip_grad_norm_(self.parameters[param_group], max_norm) + if any(any(p.grad is not None for p in g["params"]) for g in optimizer.param_groups): + grad_scaler.step(optimizer) + def optimizer_step_all(self) -> None: """Run step for all optimizers.""" for param_group, optimizer in self.optimizers.items(): diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index 2930db892..c262887d8 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -24,8 +24,8 @@ import time from dataclasses import dataclass, field from pathlib import Path from threading import Lock -from typing import Dict, List, Literal, Optional, Tuple, Type, cast - +from typing import Dict, List, Literal, Optional, Tuple, Type, cast, DefaultDict +from collections import defaultdict import torch from nerfstudio.configs.experiment_config import ExperimentConfig from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes, TrainingCallbackLocation @@ -80,8 +80,8 @@ class TrainerConfig(ExperimentConfig): """Path to checkpoint file.""" log_gradients: bool = False """Optionally log gradients during training""" - gradient_accumulation_steps: int = 1 - """Number of steps to accumulate gradients over.""" + gradient_accumulation_steps: Dict = field(default_factory=lambda: {}) + """Number of steps to accumulate gradients over. Contains a mapping of {param_group:num}""" class Trainer: @@ -118,7 +118,8 @@ class Trainer: self.mixed_precision: bool = self.config.mixed_precision self.use_grad_scaler: bool = self.mixed_precision or self.config.use_grad_scaler self.training_state: Literal["training", "paused", "completed"] = "training" - self.gradient_accumulation_steps: int = self.config.gradient_accumulation_steps + self.gradient_accumulation_steps: DefaultDict = defaultdict(lambda: 1) + self.gradient_accumulation_steps.update(self.config.gradient_accumulation_steps) if self.device == "cpu": self.mixed_precision = False @@ -459,19 +460,23 @@ class Trainer: step: Current training step. """ - self.optimizers.zero_grad_all() + needs_zero = [ + group for group in self.optimizers.parameters.keys() if step % self.gradient_accumulation_steps[group] == 0 + ] + self.optimizers.zero_grad_some(needs_zero) cpu_or_cuda_str: str = self.device.split(":")[0] cpu_or_cuda_str = "cpu" if cpu_or_cuda_str == "mps" else cpu_or_cuda_str - assert ( - self.gradient_accumulation_steps > 0 - ), f"gradient_accumulation_steps must be > 0, not {self.gradient_accumulation_steps}" - for _ in range(self.gradient_accumulation_steps): - with torch.autocast(device_type=cpu_or_cuda_str, enabled=self.mixed_precision): - _, loss_dict, metrics_dict = self.pipeline.get_train_loss_dict(step=step) - loss = functools.reduce(torch.add, loss_dict.values()) - loss /= self.gradient_accumulation_steps - self.grad_scaler.scale(loss).backward() # type: ignore - self.optimizers.optimizer_scaler_step_all(self.grad_scaler) + + with torch.autocast(device_type=cpu_or_cuda_str, enabled=self.mixed_precision): + _, loss_dict, metrics_dict = self.pipeline.get_train_loss_dict(step=step) + loss = functools.reduce(torch.add, loss_dict.values()) + self.grad_scaler.scale(loss).backward() # type: ignore + needs_step = [ + group + for group in self.optimizers.parameters.keys() + if step % self.gradient_accumulation_steps[group] == self.gradient_accumulation_steps[group] - 1 + ] + self.optimizers.optimizer_scaler_step_some(self.grad_scaler, needs_step) if self.config.log_gradients: total_grad = 0 diff --git a/nerfstudio/exporter/exporter_utils.py b/nerfstudio/exporter/exporter_utils.py index 66d658690..56b4b6445 100644 --- a/nerfstudio/exporter/exporter_utils.py +++ b/nerfstudio/exporter/exporter_utils.py @@ -28,6 +28,7 @@ import open3d as o3d import pymeshlab import torch from jaxtyping import Float +from nerfstudio.cameras.rays import RayBundle from rich.progress import BarColumn, Progress, TaskProgressColumn, TextColumn, TimeRemainingColumn from torch import Tensor @@ -130,6 +131,7 @@ def generate_point_cloud( with torch.no_grad(): ray_bundle, _ = pipeline.datamanager.next_train(0) + assert isinstance(ray_bundle, RayBundle) outputs = pipeline.model(ray_bundle) if rgb_output_name not in outputs: CONSOLE.rule("Error", style="red") diff --git a/nerfstudio/model_components/losses.py b/nerfstudio/model_components/losses.py index e90ec8e61..d46466989 100644 --- a/nerfstudio/model_components/losses.py +++ b/nerfstudio/model_components/losses.py @@ -575,6 +575,10 @@ def depth_ranking_loss(rendered_depth, gt_depth): and rendered_depth are from pixels with a radius of each other """ m = 1e-4 + if rendered_depth.shape[0] % 2 != 0: + # chop off one index + rendered_depth = rendered_depth[:-1, :] + gt_depth = gt_depth[:-1, :] dpt_diff = gt_depth[::2, :] - gt_depth[1::2, :] out_diff = rendered_depth[::2, :] - rendered_depth[1::2, :] + m differing_signs = torch.sign(dpt_diff) != torch.sign(out_diff) diff --git a/nerfstudio/models/base_model.py b/nerfstudio/models/base_model.py index d7a221744..2a434a85e 100644 --- a/nerfstudio/models/base_model.py +++ b/nerfstudio/models/base_model.py @@ -13,7 +13,7 @@ # limitations under the License. """ -Base Model implementation which takes in RayBundles +Base Model implementation which takes in RayBundles or Cameras """ from __future__ import annotations @@ -27,10 +27,11 @@ import torch from torch import nn from torch.nn import Parameter +from nerfstudio.cameras.cameras import Cameras from nerfstudio.cameras.rays import RayBundle from nerfstudio.configs.base_config import InstantiateConfig from nerfstudio.configs.config_utils import to_immutable_dict -from nerfstudio.data.scene_box import SceneBox +from nerfstudio.data.scene_box import SceneBox, OrientedBox from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes from nerfstudio.model_components.scene_colliders import NearFarCollider @@ -117,7 +118,7 @@ class Model(nn.Module): """ @abstractmethod - def get_outputs(self, ray_bundle: RayBundle) -> Dict[str, Union[torch.Tensor, List]]: + def get_outputs(self, ray_bundle: Union[RayBundle, Cameras]) -> Dict[str, Union[torch.Tensor, List]]: """Takes in a Ray Bundle and returns a dictionary of outputs. Args: @@ -128,7 +129,7 @@ class Model(nn.Module): Outputs of model. (ie. rendered colors) """ - def forward(self, ray_bundle: RayBundle) -> Dict[str, Union[torch.Tensor, List]]: + def forward(self, ray_bundle: Union[RayBundle, Cameras]) -> Dict[str, Union[torch.Tensor, List]]: """Run forward starting with a ray bundle. This outputs different things depending on the configuration of the model and whether or not the batch is provided (whether or not we are training basically) @@ -161,6 +162,18 @@ class Model(nn.Module): metrics_dict: dictionary of metrics, some of which we can use for loss """ + @torch.no_grad() + def get_outputs_for_camera(self, camera: Cameras, obb_box: Optional[OrientedBox] = None) -> Dict[str, torch.Tensor]: + """Takes in a camera, generates the raybundle, and computes the output of the model. + Assumes a ray-based model. + + Args: + camera: generates raybundle + """ + return self.get_outputs_for_camera_ray_bundle( + camera.generate_rays(camera_indices=0, keep_shape=True, obb_box=obb_box) + ) + @torch.no_grad() def get_outputs_for_camera_ray_bundle(self, camera_ray_bundle: RayBundle) -> Dict[str, torch.Tensor]: """Takes in camera parameters and computes the output of the model. diff --git a/nerfstudio/models/gaussian_splatting.py b/nerfstudio/models/gaussian_splatting.py new file mode 100644 index 000000000..fe081734e --- /dev/null +++ b/nerfstudio/models/gaussian_splatting.py @@ -0,0 +1,764 @@ +# ruff: noqa: E741 +# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +NeRF implementation that combines many recent advancements. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Tuple, Type, Union +from nerfstudio.data.scene_box import OrientedBox + +import torch +from torch.nn import Parameter +from torchmetrics.image import PeakSignalNoiseRatio +from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity +import torchvision.transforms.functional as TF + +from nerfstudio.cameras.cameras import Cameras +from gsplat._torch_impl import quat_to_rotmat +from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes, TrainingCallbackLocation +from nerfstudio.engine.optimizers import Optimizers +from nerfstudio.models.base_model import Model, ModelConfig +import math +import numpy as np +from sklearn.neighbors import NearestNeighbors +from nerfstudio.cameras.camera_optimizers import CameraOptimizer, CameraOptimizerConfig + +from gsplat.rasterize import RasterizeGaussians +from gsplat.project_gaussians import ProjectGaussians +from gsplat.sh import SphericalHarmonics, num_sh_bases +from pytorch_msssim import SSIM + +# need following import for background color override +from nerfstudio.model_components import renderers + + +def random_quat_tensor(N): + """ + Defines a random quaternion tensor of shape (N, 4) + """ + u = torch.rand(N) + v = torch.rand(N) + w = torch.rand(N) + return torch.stack( + [ + torch.sqrt(1 - u) * torch.sin(2 * math.pi * v), + torch.sqrt(1 - u) * torch.cos(2 * math.pi * v), + torch.sqrt(u) * torch.sin(2 * math.pi * w), + torch.sqrt(u) * torch.sin(2 * math.pi * w), + ], + dim=-1, + ) + + +def RGB2SH(rgb): + """ + Converts from RGB values [0,1] to the 0th spherical harmonic coefficient + """ + C0 = 0.28209479177387814 + return (rgb - 0.5) / C0 + + +def SH2RGB(sh): + """ + Converts from the 0th spherical harmonic coefficient to RGB values [0,1] + """ + C0 = 0.28209479177387814 + return sh * C0 + 0.5 + + +def projection_matrix(znear, zfar, fovx, fovy, device: Union[str, torch.device] = "cpu"): + """ + Constructs an OpenGL-style perspective projection matrix. + """ + t = znear * math.tan(0.5 * fovy) + b = -t + r = znear * math.tan(0.5 * fovx) + l = -r + n = znear + f = zfar + return torch.tensor( + [ + [2 * n / (r - l), 0.0, (r + l) / (r - l), 0.0], + [0.0, 2 * n / (t - b), (t + b) / (t - b), 0.0], + [0.0, 0.0, (f + n) / (f - n), -1.0 * f * n / (f - n)], + [0.0, 0.0, 1.0, 0.0], + ], + device=device, + ) + + +@dataclass +class GaussianSplattingModelConfig(ModelConfig): + """Gaussian Splatting Model Config""" + + _target: Type = field(default_factory=lambda: GaussianSplattingModel) + warmup_length: int = 500 + """period of steps where refinement is turned off""" + refine_every: int = 100 + """period of steps where gaussians are culled and densified""" + resolution_schedule: int = 250 + """training starts at 1/d resolution, every n steps this is doubled""" + num_downscales: int = 2 + """at the beginning, resolution is 1/2^d, where d is this number""" + cull_alpha_thresh: float = 0.1 + """threshold of opacity for culling gaussians""" + cull_scale_thresh: float = 0.5 + """threshold of scale for culling gaussians""" + reset_alpha_every: int = 30 + """Every this many refinement steps, reset the alpha""" + densify_grad_thresh: float = 0.0002 + """threshold of positional gradient norm for densifying gaussians""" + densify_size_thresh: float = 0.01 + """below this size, gaussians are *duplicated*, otherwise split""" + n_split_samples: int = 2 + """number of samples to split gaussians into""" + sh_degree_interval: int = 1000 + """every n intervals turn on another sh degree""" + cull_screen_size: float = 0.15 + """if a gaussian is more than this percent of screen space, cull it""" + split_screen_size: float = 0.05 + """if a gaussian is more than this percent of screen space, split it""" + stop_screen_size_at: int = 4000 + """stop culling/splitting at this step WRT screen size of gaussians""" + random_init: bool = False + """whether to initialize the positions uniformly randomly (not SFM points)""" + ssim_lambda: float = 0.2 + """weight of ssim loss""" + stop_split_at: int = 15000 + """stop splitting at this step""" + sh_degree: int = 4 + """maximum degree of spherical harmonics to use""" + camera_optimizer: CameraOptimizerConfig = CameraOptimizerConfig(mode="off") + """camera optimizer config""" + max_gauss_ratio: float = 10.0 + """threshold of ratio of gaussian max to min scale before applying regularization + loss from the PhysGaussian paper + """ + + +class GaussianSplattingModel(Model): + """Gaussian Splatting model + + Args: + config: Gaussian Splatting configuration to instantiate model + """ + + config: GaussianSplattingModelConfig + + def __init__(self, *args, **kwargs): + if "seed_points" in kwargs: + self.seed_pts = kwargs["seed_points"] + else: + self.seed_pts = None + super().__init__(*args, **kwargs) + + def populate_modules(self): + if self.seed_pts is not None and not self.config.random_init: + self.means = torch.nn.Parameter(self.seed_pts[0]) # (Location, Color) + else: + self.means = torch.nn.Parameter((torch.rand((500000, 3)) - 0.5) * 10) + self.xys_grad_norm = None + self.max_2Dsize = None + distances, _ = self.k_nearest_sklearn(self.means.data, 3) + distances = torch.from_numpy(distances) + # find the average of the three nearest neighbors for each point and use that as the scale + avg_dist = distances.mean(dim=-1, keepdim=True) + self.scales = torch.nn.Parameter(torch.log(avg_dist.repeat(1, 3))) + self.quats = torch.nn.Parameter(random_quat_tensor(self.num_points)) + dim_sh = num_sh_bases(self.config.sh_degree) + + if self.seed_pts is not None and not self.config.random_init: + fused_color = RGB2SH(self.seed_pts[1] / 255) + shs = torch.zeros((fused_color.shape[0], dim_sh, 3)).float().cuda() + shs[:, 0, :3] = fused_color + shs[:, 1:, 3:] = 0.0 + self.colors_all = torch.nn.Parameter(shs) + else: + colors = torch.nn.Parameter(torch.rand(self.num_points, 1, 3)) + shs_rest = torch.nn.Parameter(torch.zeros((self.num_points, dim_sh - 1, 3))) + self.colors_all = torch.nn.Parameter(torch.cat([colors, shs_rest], dim=1)) + + self.opacities = torch.nn.Parameter(torch.logit(0.1 * torch.ones(self.num_points, 1))) + + # metrics + self.psnr = PeakSignalNoiseRatio(data_range=1.0) + self.ssim = SSIM(data_range=1.0, size_average=True, channel=3) + self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True) + self.step = 0 + + self.crop_box: Optional[OrientedBox] = None + self.back_color = torch.zeros(3) + + self.camera_optimizer: CameraOptimizer = self.config.camera_optimizer.setup( + num_cameras=self.num_train_data, device="cpu" + ) + + @property + def colors(self): + return SH2RGB(self.colors_all[:, 0, :]) + + @property + def shs_rest(self): + return self.colors_all[:, 1:, :] + + def load_state_dict(self, dict, **kwargs): # type: ignore + # resize the parameters to match the new number of points + self.step = 30000 + newp = dict["means"].shape[0] + self.means = torch.nn.Parameter(torch.zeros(newp, 3, device=self.device)) + self.scales = torch.nn.Parameter(torch.zeros(newp, 3, device=self.device)) + self.quats = torch.nn.Parameter(torch.zeros(newp, 4, device=self.device)) + self.opacities = torch.nn.Parameter(torch.zeros(newp, 1, device=self.device)) + self.colors_all = torch.nn.Parameter( + torch.zeros(newp, num_sh_bases(self.config.sh_degree), 3, device=self.device) + ) + super().load_state_dict(dict, **kwargs) + + def k_nearest_sklearn(self, x: torch.Tensor, k: int): + """ + Find k-nearest neighbors using sklearn's NearestNeighbors. + x: The data tensor of shape [num_samples, num_features] + k: The number of neighbors to retrieve + """ + # Convert tensor to numpy array + x_np = x.cpu().numpy() + + # Build the nearest neighbors model + nn_model = NearestNeighbors(n_neighbors=k + 1, algorithm="auto", metric="euclidean").fit(x_np) + + # Find the k-nearest neighbors + distances, indices = nn_model.kneighbors(x_np) + + # Exclude the point itself from the result and return + return distances[:, 1:].astype(np.float32), indices[:, 1:].astype(np.float32) + + def remove_from_optim(self, optimizer, deleted_mask, new_params): + """removes the deleted_mask from the optimizer provided""" + assert len(new_params) == 1 + # assert isinstance(optimizer, torch.optim.Adam), "Only works with Adam" + + param = optimizer.param_groups[0]["params"][0] + param_state = optimizer.state[param] + del optimizer.state[param] + + # Modify the state directly without deleting and reassigning. + param_state["exp_avg"] = param_state["exp_avg"][~deleted_mask] + param_state["exp_avg_sq"] = param_state["exp_avg_sq"][~deleted_mask] + + # Update the parameter in the optimizer's param group. + del optimizer.param_groups[0]["params"][0] + del optimizer.param_groups[0]["params"] + optimizer.param_groups[0]["params"] = new_params + optimizer.state[new_params[0]] = param_state + + def dup_in_optim(self, optimizer, dup_mask, new_params, n=2): + """adds the parameters to the optimizer""" + param = optimizer.param_groups[0]["params"][0] + param_state = optimizer.state[param] + repeat_dims = (n,) + tuple(1 for _ in range(param_state["exp_avg"].dim() - 1)) + param_state["exp_avg"] = torch.cat( + [param_state["exp_avg"], torch.zeros_like(param_state["exp_avg"][dup_mask.squeeze()]).repeat(*repeat_dims)], + dim=0, + ) + param_state["exp_avg_sq"] = torch.cat( + [ + param_state["exp_avg_sq"], + torch.zeros_like(param_state["exp_avg_sq"][dup_mask.squeeze()]).repeat(*repeat_dims), + ], + dim=0, + ) + del optimizer.state[param] + optimizer.state[new_params[0]] = param_state + optimizer.param_groups[0]["params"] = new_params + del param + + def after_train(self, step: int): + with torch.no_grad(): + # keep track of a moving average of grad norms + visible_mask = (self.radii > 0).flatten() + grads = self.xys.grad.detach().norm(dim=-1) # TODO fill in + # print(f"grad norm min {grads.min().item()} max {grads.max().item()} mean {grads.mean().item()} size {grads.shape}") + if self.xys_grad_norm is None: + self.xys_grad_norm = grads + self.vis_counts = torch.ones_like(self.xys_grad_norm) + else: + assert self.vis_counts is not None + self.vis_counts[visible_mask] = self.vis_counts[visible_mask] + 1 + self.xys_grad_norm[visible_mask] = grads[visible_mask] + self.xys_grad_norm[visible_mask] + + # update the max screen size, as a ratio of number of pixels + if self.max_2Dsize is None: + self.max_2Dsize = torch.zeros_like(self.radii, dtype=torch.float32) + newradii = self.radii.detach()[visible_mask] + self.max_2Dsize[visible_mask] = torch.maximum( + self.max_2Dsize[visible_mask], newradii / float(max(self.last_size[0], self.last_size[1])) + ) + + def set_crop(self, crop_box: Optional[OrientedBox]): + self.crop_box = crop_box + + def set_background(self, back_color: torch.Tensor): + assert back_color.shape == (3,) + self.back_color = back_color + + def refinement_after(self, optimizers: Optimizers, step): + if self.step >= self.config.warmup_length: + with torch.no_grad(): + # only split/cull if we've seen every image since opacity reset + reset_interval = self.config.reset_alpha_every * self.config.refine_every + if ( + self.step < self.config.stop_split_at + and self.step % reset_interval > self.num_train_data + self.config.refine_every + ): + # then we densify + assert ( + self.xys_grad_norm is not None and self.vis_counts is not None and self.max_2Dsize is not None + ) + avg_grad_norm = ( + (self.xys_grad_norm / self.vis_counts) * 0.5 * max(self.last_size[0], self.last_size[1]) + ) + high_grads = (avg_grad_norm > self.config.densify_grad_thresh).squeeze() + splits = (self.scales.exp().max(dim=-1).values > self.config.densify_size_thresh).squeeze() + if self.step < self.config.stop_screen_size_at: + splits |= (self.max_2Dsize > self.config.split_screen_size).squeeze() + splits &= high_grads + nsamps = self.config.n_split_samples + ( + split_means, + split_colors, + split_opacities, + split_scales, + split_quats, + ) = self.split_gaussians(splits, nsamps) + + dups = (self.scales.exp().max(dim=-1).values <= self.config.densify_size_thresh).squeeze() + dups &= high_grads + dup_means, dup_colors, dup_opacities, dup_scales, dup_quats = self.dup_gaussians(dups) + self.means = Parameter(torch.cat([self.means.detach(), split_means, dup_means], dim=0)) + self.colors_all = Parameter(torch.cat([self.colors_all.detach(), split_colors, dup_colors], dim=0)) + + self.opacities = Parameter( + torch.cat([self.opacities.detach(), split_opacities, dup_opacities], dim=0) + ) + self.scales = Parameter(torch.cat([self.scales.detach(), split_scales, dup_scales], dim=0)) + self.quats = Parameter(torch.cat([self.quats.detach(), split_quats, dup_quats], dim=0)) + # append zeros to the max_2Dsize tensor + self.max_2Dsize = torch.cat( + [self.max_2Dsize, torch.zeros_like(split_scales[:, 0]), torch.zeros_like(dup_scales[:, 0])], + dim=0, + ) + split_idcs = torch.where(splits)[0] + param_groups = self.get_gaussian_param_groups() + for group, param in param_groups.items(): + self.dup_in_optim(optimizers.optimizers[group], split_idcs, param, n=nsamps) + dup_idcs = torch.where(dups)[0] + + param_groups = self.get_gaussian_param_groups() + for group, param in param_groups.items(): + self.dup_in_optim(optimizers.optimizers[group], dup_idcs, param, 1) + + # Offset all the opacity reset logic by refine_every so that we don't + # save checkpoints right when the opacity is reset (saves every 2k) + if self.step % reset_interval > self.num_train_data + self.config.refine_every: + # then cull + deleted_mask = self.cull_gaussians() + param_groups = self.get_gaussian_param_groups() + for group, param in param_groups.items(): + self.remove_from_optim(optimizers.optimizers[group], deleted_mask, param) + + if self.step % reset_interval == self.config.refine_every: + reset_value = self.config.cull_alpha_thresh * 0.8 + self.opacities.data = torch.full_like( + self.opacities.data, torch.logit(torch.tensor(reset_value)).item() + ) + # reset the exp of optimizer + optim = optimizers.optimizers["opacity"] + param = optim.param_groups[0]["params"][0] + param_state = optim.state[param] + param_state["exp_avg"] = torch.zeros_like(param_state["exp_avg"]) + param_state["exp_avg_sq"] = torch.zeros_like(param_state["exp_avg_sq"]) + self.xys_grad_norm = None + self.vis_counts = None + self.max_2Dsize = None + + def cull_gaussians(self): + """ + This function deletes gaussians with under a certain opacity threshold + """ + n_bef = self.num_points + # cull transparent ones + culls = (torch.sigmoid(self.opacities) < self.config.cull_alpha_thresh).squeeze() + if self.step > self.config.refine_every * self.config.reset_alpha_every: + # cull huge ones + toobigs = (torch.exp(self.scales).max(dim=-1).values > self.config.cull_scale_thresh).squeeze() + culls = culls | toobigs + if self.step < self.config.stop_screen_size_at: + # cull big screen space + assert self.max_2Dsize is not None + culls = culls | (self.max_2Dsize > self.config.cull_screen_size).squeeze() + self.means = Parameter(self.means[~culls].detach()) + self.scales = Parameter(self.scales[~culls].detach()) + self.quats = Parameter(self.quats[~culls].detach()) + self.colors_all = Parameter(self.colors_all[~culls].detach()) + self.opacities = Parameter(self.opacities[~culls].detach()) + + print(f"Culled {n_bef - self.num_points} gaussians") + return culls + + def split_gaussians(self, split_mask, samps): + """ + This function splits gaussians that are too large + """ + + n_splits = split_mask.sum().item() + print(f"Splitting {split_mask.sum().item()/self.num_points} gaussians: {n_splits}/{self.num_points}") + centered_samples = torch.randn((samps * n_splits, 3), device=self.device) # Nx3 of axis-aligned scales + scaled_samples = ( + torch.exp(self.scales[split_mask].repeat(samps, 1)) * centered_samples + ) # how these scales are rotated + quats = self.quats[split_mask] / self.quats[split_mask].norm(dim=-1, keepdim=True) # normalize them first + rots = quat_to_rotmat(quats.repeat(samps, 1)) # how these scales are rotated + rotated_samples = torch.bmm(rots, scaled_samples[..., None]).squeeze() + new_means = rotated_samples + self.means[split_mask].repeat(samps, 1) + # step 2, sample new colors + new_colors_all = self.colors_all[split_mask].repeat(samps, 1, 1) + # step 3, sample new opacities + new_opacities = self.opacities[split_mask].repeat(samps, 1) + # step 4, sample new scales + size_fac = 1.6 + new_scales = torch.log(torch.exp(self.scales[split_mask]) / size_fac).repeat(samps, 1) + self.scales[split_mask] = torch.log(torch.exp(self.scales[split_mask]) / size_fac) + # step 5, sample new quats + new_quats = self.quats[split_mask].repeat(samps, 1) + return new_means, new_colors_all, new_opacities, new_scales, new_quats + + def dup_gaussians(self, dup_mask): + """ + This function duplicates gaussians that are too small + """ + n_dups = dup_mask.sum().item() + print(f"Duplicating {dup_mask.sum().item()/self.num_points} gaussians: {n_dups}/{self.num_points}") + dup_means = self.means[dup_mask] + dup_colors = self.colors_all[dup_mask] + dup_opacities = self.opacities[dup_mask] + dup_scales = self.scales[dup_mask] + dup_quats = self.quats[dup_mask] + return dup_means, dup_colors, dup_opacities, dup_scales, dup_quats + + @property + def num_points(self): + return self.means.shape[0] + + def get_training_callbacks( + self, training_callback_attributes: TrainingCallbackAttributes + ) -> List[TrainingCallback]: + cbs = [] + cbs.append(TrainingCallback([TrainingCallbackLocation.BEFORE_TRAIN_ITERATION], self.step_cb)) + # The order of these matters + cbs.append( + TrainingCallback( + [TrainingCallbackLocation.AFTER_TRAIN_ITERATION], + self.after_train, + ) + ) + cbs.append( + TrainingCallback( + [TrainingCallbackLocation.AFTER_TRAIN_ITERATION], + self.refinement_after, + update_every_num_iters=self.config.refine_every, + args=[training_callback_attributes.optimizers], + ) + ) + return cbs + + def step_cb(self, step): + self.step = step + + def get_gaussian_param_groups(self) -> Dict[str, List[Parameter]]: + return { + "xyz": [self.means], + "color": [self.colors_all], + "opacity": [self.opacities], + "scaling": [self.scales], + "rotation": [self.quats], + } + + def get_param_groups(self) -> Dict[str, List[Parameter]]: + """Obtain the parameter groups for the optimizers + + Returns: + Mapping of different parameter groups + """ + gps = self.get_gaussian_param_groups() + # add camera optimizer param groups + self.camera_optimizer.get_param_groups(gps) + return gps + + def _get_downscale_factor(self): + if self.training: + return 2 ** max((self.config.num_downscales - self.step // self.config.resolution_schedule), 0) + else: + return 1 + + def get_outputs(self, camera: Cameras) -> Dict[str, Union[torch.Tensor, List]]: + """Takes in a Ray Bundle and returns a dictionary of outputs. + + Args: + ray_bundle: Input bundle of rays. This raybundle should have all the + needed information to compute the outputs. + + Returns: + Outputs of model. (ie. rendered colors) + """ + if not isinstance(camera, Cameras): + print("Called get_outputs with not a camera") + return {} + assert camera.shape[0] == 1, "Only one camera at a time" + if self.training: + # currently relies on the branch vickie/camera-grads + self.camera_optimizer.apply_to_camera(camera) + if self.training: + background = torch.rand(3, device=self.device) + else: + # logic for setting the background of the scene + if renderers.BACKGROUND_COLOR_OVERRIDE is not None: + background = renderers.BACKGROUND_COLOR_OVERRIDE + else: + background = self.back_color.to(self.device) + if self.crop_box is not None and not self.training: + crop_ids = self.crop_box.within(self.means).squeeze() + if crop_ids.sum() == 0: + return {"rgb": background.repeat(int(camera.height.item()), int(camera.width.item()), 1)} + else: + crop_ids = None + camera_downscale = self._get_downscale_factor() + camera.rescale_output_resolution(1 / camera_downscale) + # shift the camera to center of scene looking at center + R = camera.camera_to_worlds[0, :3, :3] # 3 x 3 + T = camera.camera_to_worlds[0, :3, 3:4] # 3 x 1 + # flip the z and y axes to align with gsplat conventions + R_edit = torch.diag(torch.tensor([1, -1, -1], device="cuda", dtype=R.dtype)) + R = R @ R_edit + # analytic matrix inverse to get world2camera matrix + R_inv = R.T + T_inv = -R_inv @ T + viewmat = torch.eye(4, device=R.device, dtype=R.dtype) + viewmat[:3, :3] = R_inv + viewmat[:3, 3:4] = T_inv + # calculate the FOV of the camera given fx and fy, width and height + cx = camera.cx.item() + cy = camera.cy.item() + fovx = 2 * math.atan(camera.width / (2 * camera.fx)) + fovy = 2 * math.atan(camera.height / (2 * camera.fy)) + W, H = camera.width.item(), camera.height.item() + self.last_size = (H, W) + projmat = projection_matrix(0.001, 1000, fovx, fovy, device=self.device) + BLOCK_X, BLOCK_Y = 16, 16 + tile_bounds = ( + (W + BLOCK_X - 1) // BLOCK_X, + (H + BLOCK_Y - 1) // BLOCK_Y, + 1, + ) + + if crop_ids is not None: + opacities_crop = self.opacities[crop_ids] + means_crop = self.means[crop_ids] + colors_crop = self.colors_all[crop_ids] + scales_crop = self.scales[crop_ids] + quats_crop = self.quats[crop_ids] + else: + opacities_crop = self.opacities + means_crop = self.means + colors_crop = self.colors_all + scales_crop = self.scales + quats_crop = self.quats + self.xys, depths, self.radii, conics, num_tiles_hit, _ = ProjectGaussians.apply( # type: ignore + means_crop, + torch.exp(scales_crop), + 1, + quats_crop / quats_crop.norm(dim=-1, keepdim=True), + viewmat.squeeze()[:3, :], + projmat.squeeze() @ viewmat.squeeze(), + camera.fx.item(), + camera.fy.item(), + cx, + cy, + H, + W, + tile_bounds, + ) + if (self.radii).sum() == 0: + return {"rgb": background.repeat(int(camera.height.item()), int(camera.width.item()), 1)} + + # Important to allow xys grads to populate properly + if self.training: + self.xys.retain_grad() + if self.config.sh_degree > 0: + viewdirs = means_crop.detach() - camera.camera_to_worlds.detach()[..., :3, 3] # (N, 3) + viewdirs = viewdirs / viewdirs.norm(dim=-1, keepdim=True) + n = min(self.step // self.config.sh_degree_interval, self.config.sh_degree) + rgbs = SphericalHarmonics.apply(n, viewdirs, colors_crop) + rgbs = torch.clamp(rgbs + 0.5, 0.0, 1.0) # type: ignore + else: + rgbs = self.get_colors.squeeze() # (N, 3) + rgbs = torch.sigmoid(rgbs) + rgb = RasterizeGaussians.apply( + self.xys, + depths, + self.radii, + conics, + num_tiles_hit, + rgbs, + torch.sigmoid(opacities_crop), + H, + W, + background, + ) + depth_im = None + if not self.training: + depth_im = RasterizeGaussians.apply( # type: ignore + self.xys, + depths, + self.radii, + conics, + num_tiles_hit, + depths[:, None].repeat(1, 3), + torch.sigmoid(opacities_crop), + H, + W, + torch.ones(3, device=self.device) * 10, + )[..., 0:1] + # rescale the camera back to original dimensions + camera.rescale_output_resolution(camera_downscale) + return {"rgb": rgb, "depth": depth_im} # type: ignore + + def get_metrics_dict(self, outputs, batch) -> Dict[str, torch.Tensor]: + """Compute and returns metrics. + + Args: + outputs: the output to compute loss dict to + batch: ground truth batch corresponding to outputs + """ + d = self._get_downscale_factor() + if d > 1: + newsize = [batch["image"].shape[0] // d, batch["image"].shape[1] // d] + gt_img = TF.resize(batch["image"].permute(2, 0, 1), newsize, antialias=None).permute(1, 2, 0) + else: + gt_img = batch["image"] + metrics_dict = {} + gt_rgb = gt_img.to(self.device) # RGB or RGBA image + predicted_rgb = outputs["rgb"] + metrics_dict["psnr"] = self.psnr(predicted_rgb, gt_rgb) + + self.camera_optimizer.get_metrics_dict(metrics_dict) + metrics_dict["gaussian_count"] = self.num_points + return metrics_dict + + def get_loss_dict(self, outputs, batch, metrics_dict=None) -> Dict[str, torch.Tensor]: + """Computes and returns the losses dict. + + Args: + outputs: the output to compute loss dict to + batch: ground truth batch corresponding to outputs + metrics_dict: dictionary of metrics, some of which we can use for loss + """ + d = self._get_downscale_factor() + if d > 1: + newsize = [batch["image"].shape[0] // d, batch["image"].shape[1] // d] + gt_img = TF.resize(batch["image"].permute(2, 0, 1), newsize, antialias=None).permute(1, 2, 0) + else: + gt_img = batch["image"] + Ll1 = torch.abs(gt_img - outputs["rgb"]).mean() + simloss = 1 - self.ssim(gt_img.permute(2, 0, 1)[None, ...], outputs["rgb"].permute(2, 0, 1)[None, ...]) + if self.step % 10 == 0: + # Before, we made split sh and colors onto different optimizer, with shs having a low learning rate + # This is slow, instead we apply a regularization every few steps + sh_reg = self.colors_all[:, 1:, :].norm(dim=1).mean() + scale_exp = torch.exp(self.scales) + scale_reg = ( + torch.maximum( + scale_exp.amax(dim=-1) / scale_exp.amin(dim=-1), torch.tensor(self.config.max_gauss_ratio) + ) + - self.config.max_gauss_ratio + ) + scale_reg = 0.1 * scale_reg.mean() + else: + sh_reg = torch.tensor(0.0).to(self.device) + scale_reg = torch.tensor(0.0).to(self.device) + return { + "main_loss": (1 - self.config.ssim_lambda) * Ll1 + self.config.ssim_lambda * simloss, + "sh_reg": sh_reg, + "scale_reg": scale_reg, + } + + @torch.no_grad() + def get_outputs_for_camera(self, camera: Cameras, obb_box: Optional[OrientedBox] = None) -> Dict[str, torch.Tensor]: + """Takes in a camera, generates the raybundle, and computes the output of the model. + Overridden for a camera-based gaussian model. + + Args: + camera: generates raybundle + """ + assert camera is not None, "must provide camera to gaussian model" + self.set_crop(obb_box) + outs = self.get_outputs(camera.to(self.device)) + return outs # type: ignore + + def get_image_metrics_and_images( + self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor] + ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: + """Writes the test image outputs. + + Args: + image_idx: Index of the image. + step: Current step. + batch: Batch of data. + outputs: Outputs of the model. + + Returns: + A dictionary of metrics. + """ + d = self._get_downscale_factor() + if d > 1: + newsize = [batch["image"].shape[0] // d, batch["image"].shape[1] // d] + gt_img = TF.resize(batch["image"].permute(2, 0, 1), newsize, antialias=None).permute(1, 2, 0) + predicted_rgb = TF.resize(outputs["rgb"].permute(2, 0, 1), newsize, antialias=None).permute(1, 2, 0) + else: + gt_img = batch["image"] + predicted_rgb = outputs["rgb"] + + gt_rgb = gt_img.to(self.device) + + combined_rgb = torch.cat([gt_rgb, predicted_rgb], dim=1) + + # Switch images from [H, W, C] to [1, C, H, W] for metrics computations + gt_rgb = torch.moveaxis(gt_rgb, -1, 0)[None, ...] + predicted_rgb = torch.moveaxis(predicted_rgb, -1, 0)[None, ...] + + psnr = self.psnr(gt_rgb, predicted_rgb) + ssim = self.ssim(gt_rgb, predicted_rgb) + lpips = self.lpips(gt_rgb, predicted_rgb) + + # all of these metrics will be logged as scalars + metrics_dict = {"psnr": float(psnr.item()), "ssim": float(ssim)} # type: ignore + metrics_dict["lpips"] = float(lpips) + + images_dict = {"img": combined_rgb} + + return metrics_dict, images_dict diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py index 345a39d4b..b75adc6ad 100644 --- a/nerfstudio/pipelines/base_pipeline.py +++ b/nerfstudio/pipelines/base_pipeline.py @@ -26,18 +26,11 @@ from typing import Any, Dict, List, Literal, Mapping, Optional, Tuple, Type, Uni import torch import torch.distributed as dist -from PIL import Image -from rich.progress import ( - BarColumn, - MofNCompleteColumn, - Progress, - TextColumn, - TimeElapsedColumn, -) +from rich.progress import BarColumn, MofNCompleteColumn, Progress, TextColumn, TimeElapsedColumn from torch import nn +from torch.cuda.amp.grad_scaler import GradScaler from torch.nn import Parameter from torch.nn.parallel import DistributedDataParallel as DDP -from torch.cuda.amp.grad_scaler import GradScaler from nerfstudio.configs import base_config as cfg from nerfstudio.data.datamanagers.base_datamanager import ( @@ -46,6 +39,7 @@ from nerfstudio.data.datamanagers.base_datamanager import ( VanillaDataManager, ) from nerfstudio.data.datamanagers.parallel_datamanager import ParallelDataManager +from nerfstudio.data.datamanagers.full_images_datamanager import FullImageDatamanager from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes from nerfstudio.models.base_model import Model, ModelConfig from nerfstudio.utils import profiler @@ -264,6 +258,15 @@ class VanillaPipeline(Pipeline): self.datamanager: DataManager = config.datamanager.setup( device=device, test_mode=test_mode, world_size=world_size, local_rank=local_rank ) + # TODO make cleaner + seed_pts = None + if ( + hasattr(self.datamanager, "train_dataparser_outputs") + and "points3D_xyz" in self.datamanager.train_dataparser_outputs.metadata + ): + pts = self.datamanager.train_dataparser_outputs.metadata["points3D_xyz"] + pts_rgb = self.datamanager.train_dataparser_outputs.metadata["points3D_rgb"] + seed_pts = (pts, pts_rgb) self.datamanager.to(device) # TODO(ethan): get rid of scene_bounds from the model assert self.datamanager.train_dataset is not None, "Missing input dataset" @@ -274,6 +277,7 @@ class VanillaPipeline(Pipeline): metadata=self.datamanager.train_dataset.metadata, device=device, grad_scaler=grad_scaler, + seed_points=seed_pts, ) self.model.to(device) @@ -335,13 +339,11 @@ class VanillaPipeline(Pipeline): step: current iteration step """ self.eval() - image_idx, camera_ray_bundle, batch = self.datamanager.next_eval_image(step) - outputs = self.model.get_outputs_for_camera_ray_bundle(camera_ray_bundle) + camera, batch = self.datamanager.next_eval_image(step) + outputs = self.model.get_outputs_for_camera(camera) metrics_dict, images_dict = self.model.get_image_metrics_and_images(outputs, batch) - assert "image_idx" not in metrics_dict - metrics_dict["image_idx"] = image_idx assert "num_rays" not in metrics_dict - metrics_dict["num_rays"] = len(camera_ray_bundle) + metrics_dict["num_rays"] = (camera.height * camera.width * camera.size).item() self.train() return metrics_dict, images_dict @@ -361,7 +363,7 @@ class VanillaPipeline(Pipeline): """ self.eval() metrics_dict_list = [] - assert isinstance(self.datamanager, (VanillaDataManager, ParallelDataManager)) + assert isinstance(self.datamanager, (VanillaDataManager, ParallelDataManager, FullImageDatamanager)) num_images = len(self.datamanager.fixed_indices_eval_dataloader) with Progress( TextColumn("[progress.description]{task.description}"), @@ -371,26 +373,21 @@ class VanillaPipeline(Pipeline): transient=True, ) as progress: task = progress.add_task("[green]Evaluating all eval images...", total=num_images) - for camera_ray_bundle, batch in self.datamanager.fixed_indices_eval_dataloader: + for camera, batch in self.datamanager.fixed_indices_eval_dataloader: # time this the following line inner_start = time() - height, width = camera_ray_bundle.shape + outputs = self.model.get_outputs_for_camera(camera=camera) + height, width = camera.height, camera.width num_rays = height * width - outputs = self.model.get_outputs_for_camera_ray_bundle(camera_ray_bundle) - metrics_dict, images_dict = self.model.get_image_metrics_and_images(outputs, batch) - + metrics_dict, _ = self.model.get_image_metrics_and_images(outputs, batch) if output_path is not None: - camera_indices = camera_ray_bundle.camera_indices - assert camera_indices is not None - for key, val in images_dict.items(): - Image.fromarray((val * 255).byte().cpu().numpy()).save( - output_path / "{0:06d}-{1}.jpg".format(int(camera_indices[0, 0, 0]), key) - ) + raise NotImplementedError("Saving images is not implemented yet") + assert "num_rays_per_sec" not in metrics_dict - metrics_dict["num_rays_per_sec"] = num_rays / (time() - inner_start) + metrics_dict["num_rays_per_sec"] = (num_rays / (time() - inner_start)).item() fps_str = "fps" assert fps_str not in metrics_dict - metrics_dict[fps_str] = metrics_dict["num_rays_per_sec"] / (height * width) + metrics_dict[fps_str] = (metrics_dict["num_rays_per_sec"] / (height * width)).item() metrics_dict_list.append(metrics_dict) progress.advance(task) # average the metrics list diff --git a/nerfstudio/scripts/exporter.py b/nerfstudio/scripts/exporter.py index b104597ab..9cebd896c 100644 --- a/nerfstudio/scripts/exporter.py +++ b/nerfstudio/scripts/exporter.py @@ -37,15 +37,10 @@ from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager from nerfstudio.data.datamanagers.parallel_datamanager import ParallelDataManager from nerfstudio.data.scene_box import OrientedBox from nerfstudio.exporter import texture_utils, tsdf_utils -from nerfstudio.exporter.exporter_utils import ( - collect_camera_poses, - generate_point_cloud, - get_mesh_from_filename, -) -from nerfstudio.exporter.marching_cubes import ( - generate_mesh_with_multires_marching_cubes, -) +from nerfstudio.exporter.exporter_utils import collect_camera_poses, generate_point_cloud, get_mesh_from_filename +from nerfstudio.exporter.marching_cubes import generate_mesh_with_multires_marching_cubes from nerfstudio.fields.sdf_field import SDFField +from nerfstudio.models.gaussian_splatting import GaussianSplattingModel from nerfstudio.pipelines.base_pipeline import Pipeline, VanillaPipeline from nerfstudio.utils.eval_utils import eval_setup from nerfstudio.utils.rich_utils import CONSOLE @@ -126,6 +121,9 @@ class ExportPointCloud(Exporter): """Number of rays to evaluate per batch. Decrease if you run out of memory.""" std_ratio: float = 10.0 """Threshold based on STD of the average distances across the point cloud to remove outliers.""" + save_world_frame: bool = True + """If true, saves in the frame of the transform.json file, if false saves in the frame of the scaled + dataparser transform""" def main(self) -> None: """Export point cloud.""" @@ -162,6 +160,17 @@ class ExportPointCloud(Exporter): crop_obb=crop_obb, std_ratio=self.std_ratio, ) + if self.save_world_frame: + # apply the inverse dataparser transform to the point cloud + points = np.asarray(pcd.points) + poses = np.eye(4, dtype=np.float32)[None, ...].repeat(points.shape[0], axis=0)[:, :3, :] + poses[:, :3, 3] = points + poses = pipeline.datamanager.train_dataparser_outputs.transform_poses_to_original_space( + torch.from_numpy(poses) + ) + points = poses[:, :3, 3].numpy() + pcd.points = o3d.utility.Vector3dVector(points) + torch.cuda.empty_cache() CONSOLE.print(f"[bold green]:white_check_mark: Generated {pcd}") @@ -469,6 +478,58 @@ class ExportCameraPoses(Exporter): CONSOLE.print(f"[bold green]:white_check_mark: Saved poses to {output_file_path}") +@dataclass +class ExportGaussianSplat(Exporter): + """ + Export 3D Gaussian Splatting model to a .ply + """ + + def main(self) -> None: + if not self.output_dir.exists(): + self.output_dir.mkdir(parents=True) + + _, pipeline, _, _ = eval_setup(self.load_config) + + assert isinstance(pipeline.model, GaussianSplattingModel) + + model: GaussianSplattingModel = pipeline.model + + filename = self.output_dir / "point_cloud.ply" + + map_to_tensors = {} + + with torch.no_grad(): + positions = model.means.cpu().numpy() + map_to_tensors["positions"] = o3d.core.Tensor(positions, o3d.core.float32) + map_to_tensors["normals"] = o3d.core.Tensor(np.zeros_like(positions), o3d.core.float32) + + colors = model.colors.data.cpu().numpy() + map_to_tensors["colors"] = (colors * 255).astype(np.uint8) + for i in range(colors.shape[1]): + map_to_tensors[f"f_dc_{i}"] = colors[:, i : i + 1] + + shs = model.shs_rest.data.cpu().numpy() + if model.config.sh_degree > 0: + shs = shs.reshape((colors.shape[0], -1, 1)) + for i in range(shs.shape[-1]): + map_to_tensors[f"f_rest_{i}"] = shs[:, i] + + map_to_tensors["opacity"] = model.opacities.data.cpu().numpy() + + scales = model.scales.data.cpu().unsqueeze(-1).numpy() + for i in range(3): + map_to_tensors[f"scale_{i}"] = scales[:, i] + + quats = model.quats.data.cpu().unsqueeze(-1).numpy() + + for i in range(4): + map_to_tensors[f"rot_{i}"] = quats[:, i] + + pcd = o3d.t.geometry.PointCloud(map_to_tensors) + + o3d.t.io.write_point_cloud(str(filename), pcd) + + Commands = tyro.conf.FlagConversionOff[ Union[ Annotated[ExportPointCloud, tyro.conf.subcommand(name="pointcloud")], @@ -476,6 +537,7 @@ Commands = tyro.conf.FlagConversionOff[ Annotated[ExportPoissonMesh, tyro.conf.subcommand(name="poisson")], Annotated[ExportMarchingCubesMesh, tyro.conf.subcommand(name="marching-cubes")], Annotated[ExportCameraPoses, tyro.conf.subcommand(name="cameras")], + Annotated[ExportGaussianSplat, tyro.conf.subcommand(name="gaussian-splat")], ] ] diff --git a/nerfstudio/scripts/render.py b/nerfstudio/scripts/render.py index 79cc6c7bd..114e4195e 100644 --- a/nerfstudio/scripts/render.py +++ b/nerfstudio/scripts/render.py @@ -156,7 +156,6 @@ def _render_trajectory_video( obb_box = None if crop_data is not None: obb_box = crop_data.obb - camera_ray_bundle = cameras.generate_rays(camera_indices=camera_idx, obb_box=obb_box) max_dist, max_idx = -1, -1 true_max_dist, true_max_idx = -1, -1 @@ -206,10 +205,14 @@ def _render_trajectory_video( with renderers.background_color_override_context( crop_data.background_color.to(pipeline.device) ), torch.no_grad(): - outputs = pipeline.model.get_outputs_for_camera_ray_bundle(camera_ray_bundle) + outputs = pipeline.model.get_outputs_for_camera( + cameras[camera_idx : camera_idx + 1], obb_box=obb_box + ) else: with torch.no_grad(): - outputs = pipeline.model.get_outputs_for_camera_ray_bundle(camera_ray_bundle) + outputs = pipeline.model.get_outputs_for_camera( + cameras[camera_idx : camera_idx + 1], obb_box=obb_box + ) render_image = [] for rendered_output_name in rendered_output_names: @@ -679,7 +682,7 @@ class SpiralRender(BaseRender): ), ) steps = int(self.frame_rate * self.seconds) - camera_start = pipeline.datamanager.eval_dataloader.get_camera(image_idx=0).flatten() + camera_start, _ = pipeline.datamanager.eval_dataloader.get_camera(image_idx=0) camera_path = get_spiral_path(camera_start, steps=steps, radius=self.radius) _render_trajectory_video( @@ -788,10 +791,9 @@ class DatasetRender(BaseRender): TimeRemainingColumn(elapsed_when_finished=False, compact=False), TimeElapsedColumn(), ) as progress: - for camera_idx, (ray_bundle, batch) in enumerate(progress.track(dataloader, total=len(dataset))): - ray_bundle: RayBundle + for camera_idx, (camera, batch) in enumerate(progress.track(dataloader, total=len(dataset))): with torch.no_grad(): - outputs = pipeline.model.get_outputs_for_camera_ray_bundle(ray_bundle) + outputs = pipeline.model.get_outputs_for_camera(camera) gt_batch = batch.copy() gt_batch["rgb"] = gt_batch.pop("image") diff --git a/nerfstudio/scripts/viewer/run_viewer.py b/nerfstudio/scripts/viewer/run_viewer.py index de210a30d..b1b87c42c 100644 --- a/nerfstudio/scripts/viewer/run_viewer.py +++ b/nerfstudio/scripts/viewer/run_viewer.py @@ -104,7 +104,7 @@ def _start_viewer(config: TrainerConfig, pipeline: Pipeline, step: int): viewer_state = ViewerBetaState( config.viewer, log_filename=viewer_log_path, - datapath=base_dir, + datapath=pipeline.datamanager.get_datapath(), pipeline=pipeline, share=config.viewer.make_share_url, ) diff --git a/nerfstudio/viewer/server/render_state_machine.py b/nerfstudio/viewer/server/render_state_machine.py index 390390d48..753300521 100644 --- a/nerfstudio/viewer/server/render_state_machine.py +++ b/nerfstudio/viewer/server/render_state_machine.py @@ -24,6 +24,7 @@ import torch from nerfstudio.cameras.cameras import Cameras from nerfstudio.model_components.renderers import background_color_override_context +from nerfstudio.models.gaussian_splatting import GaussianSplattingModel from nerfstudio.utils import colormaps, writer from nerfstudio.utils.writer import GLOBAL_BUFFER, EventName, TimeWriter from nerfstudio.viewer.server import viewer_utils @@ -127,7 +128,14 @@ class RenderStateMachine(threading.Thread): assert camera is not None, "render called before viewer connected" with self.viewer.train_lock if self.viewer.train_lock is not None else contextlib.nullcontext(): - camera_ray_bundle = camera.generate_rays(camera_indices=0, aabb_box=self.viewer.get_model().render_aabb) + # TODO jake-austin: Make this check whether the model inherits from a camera based model or a ray based model + # TODO Zhuoyang: First made some dummy judgements, need to be fixed later + isGaussianSplattingModel = isinstance(self.viewer.get_model(), GaussianSplattingModel) + if isGaussianSplattingModel: + # TODO fix me before ship + camera_ray_bundle = camera.generate_rays(camera_indices=0, aabb_box=self.viewer.get_model().render_aabb) + else: + camera_ray_bundle = camera.generate_rays(camera_indices=0, aabb_box=self.viewer.get_model().render_aabb) with TimeWriter(None, None, write=False) as vis_t: self.viewer.get_model().eval() @@ -142,12 +150,21 @@ class RenderStateMachine(threading.Thread): device=self.viewer.get_model().device, ) with background_color_override_context(background_color), torch.no_grad(): - outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) + if isGaussianSplattingModel: + outputs = self.viewer.get_model().get_outputs_for_camera(camera) + else: + outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) else: with torch.no_grad(): - outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) + if isGaussianSplattingModel: + outputs = self.viewer.get_model().get_outputs_for_camera(camera) + else: + outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) self.viewer.get_model().train() - num_rays = len(camera_ray_bundle) + if True: + num_rays = (camera.height * camera.width).item() + else: + num_rays = len(camera_ray_bundle) render_time = vis_t.duration if writer.is_initialized(): writer.put_time( diff --git a/nerfstudio/viewer_beta/export_panel.py b/nerfstudio/viewer_beta/export_panel.py index 425e10a19..a5564327a 100644 --- a/nerfstudio/viewer_beta/export_panel.py +++ b/nerfstudio/viewer_beta/export_panel.py @@ -81,6 +81,11 @@ def populate_point_cloud_tab( ) -> None: server.add_gui_markdown("Render depth, project to an oriented point cloud, and filter. ") num_points = server.add_gui_number("# Points", initial_value=1_000_000, min=1, max=None, step=1) + world_frame = server.add_gui_checkbox( + "Save in world frame", + True, + hint="Save the point cloud in the transforms.json frame, rather than scaled scene frame", + ) remove_outliers = server.add_gui_checkbox("Remove outliers", True) normals = server.add_gui_dropdown( "Normals", @@ -104,6 +109,7 @@ def populate_point_cloud_tab( f"--remove-outliers {remove_outliers.value}", f"--normal-method {normals.value}", f"--use_bounding_box {control_panel.crop_viewport}", + f"--save-world-frame {world_frame.value}", get_crop_string(control_panel.crop_obb), ] ) diff --git a/nerfstudio/viewer_beta/render_state_machine.py b/nerfstudio/viewer_beta/render_state_machine.py index 53c73c34e..849cf3d4e 100644 --- a/nerfstudio/viewer_beta/render_state_machine.py +++ b/nerfstudio/viewer_beta/render_state_machine.py @@ -27,6 +27,8 @@ from nerfstudio.utils import colormaps, writer from nerfstudio.utils.writer import GLOBAL_BUFFER, EventName, TimeWriter from nerfstudio.viewer.server import viewer_utils from nerfstudio.viewer_beta.utils import CameraState, get_camera +from nerfstudio.models.gaussian_splatting import GaussianSplattingModel +from nerfstudio.cameras.cameras import Cameras if TYPE_CHECKING: from nerfstudio.viewer_beta.viewer import Viewer @@ -125,11 +127,18 @@ class RenderStateMachine(threading.Thread): camera = get_camera(camera_state, image_height, image_width) camera = camera.to(self.viewer.get_model().device) + assert isinstance(camera, Cameras) assert camera is not None, "render called before viewer connected" with TimeWriter(None, None, write=False) as vis_t: with self.viewer.train_lock if self.viewer.train_lock is not None else contextlib.nullcontext(): - camera_ray_bundle = camera.generate_rays(camera_indices=0, obb_box=obb) + if isinstance(self.viewer.get_model(), GaussianSplattingModel): + color = self.viewer.control_panel.background_color + background_color = torch.tensor( + [color[0] / 255.0, color[1] / 255.0, color[2] / 255.0], + device=self.viewer.get_model().device, + ) + self.viewer.get_model().set_background(background_color) self.viewer.get_model().eval() step = self.viewer.step try: @@ -145,21 +154,27 @@ class RenderStateMachine(threading.Thread): with background_color_override_context( background_color ), torch.no_grad(), viewer_utils.SetTrace(self.check_interrupt): - outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) + outputs = self.viewer.get_model().get_outputs_for_camera(camera, obb_box=obb) else: with torch.no_grad(), viewer_utils.SetTrace(self.check_interrupt): - outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) + outputs = self.viewer.get_model().get_outputs_for_camera(camera, obb_box=obb) except viewer_utils.IOChangeException: self.viewer.get_model().train() raise self.viewer.get_model().train() - num_rays = len(camera_ray_bundle) + num_rays = (camera.height * camera.width).item() if self.viewer.control_panel.layer_depth: - # convert to z_depth if depth compositing is enabled - R = camera.camera_to_worlds[0:3, 0:3].T - pts = camera_ray_bundle.directions * outputs["depth"] - pts = (R @ (pts.view(-1, 3).T)).T.view(*camera_ray_bundle.directions.shape) - outputs["gl_z_buf_depth"] = -pts[..., 2:3] # negative z axis is the coordinate convention + if isinstance(self.viewer.get_model(), GaussianSplattingModel): + # TODO: sending depth at high resolution lags the network a lot, figure out how to do this more efficiently + # outputs["gl_z_buf_depth"] = outputs["depth"] + pass + else: + # convert to z_depth if depth compositing is enabled + R = camera.camera_to_worlds[0, 0:3, 0:3].T + camera_ray_bundle = camera.generate_rays(camera_indices=0, obb_box=obb) + pts = camera_ray_bundle.directions * outputs["depth"] + pts = (R @ (pts.view(-1, 3).T)).T.view(*camera_ray_bundle.directions.shape) + outputs["gl_z_buf_depth"] = -pts[..., 2:3] # negative z axis is the coordinate convention render_time = vis_t.duration if writer.is_initialized() and render_time != 0: writer.put_time( @@ -187,7 +202,7 @@ class RenderStateMachine(threading.Thread): except viewer_utils.IOChangeException: # if we got interrupted, don't send the output to the viewer continue - self._send_output_to_viewer(outputs) + self._send_output_to_viewer(outputs, static_render=(action.action in ["static", "step"])) def check_interrupt(self, frame, event, arg): """Raises interrupt when flag has been set and not already on lowest resolution. @@ -199,7 +214,7 @@ class RenderStateMachine(threading.Thread): raise viewer_utils.IOChangeException return self.check_interrupt - def _send_output_to_viewer(self, outputs: Dict[str, Any]): + def _send_output_to_viewer(self, outputs: Dict[str, Any], static_render: bool = True): """Chooses the correct output and sends it to the viewer Args: @@ -239,11 +254,11 @@ class RenderStateMachine(threading.Thread): depth = ( outputs["gl_z_buf_depth"].cpu().numpy() * self.viser_scale_ratio if "gl_z_buf_depth" in outputs else None ) - + jpg_quality = self.viewer.config.jpeg_quality if static_render else 40 self.client.set_background_image( selected_output.cpu().numpy(), format=self.viewer.config.image_format, - jpeg_quality=self.viewer.config.jpeg_quality, + jpeg_quality=jpg_quality, depth=depth, ) res = f"{selected_output.shape[0]}x{selected_output.shape[1]}px" diff --git a/nerfstudio/viewer_beta/utils.py b/nerfstudio/viewer_beta/utils.py index a3dd7b1c2..3614fce41 100644 --- a/nerfstudio/viewer_beta/utils.py +++ b/nerfstudio/viewer_beta/utils.py @@ -65,7 +65,7 @@ def get_camera( cx=pp_w, cy=pp_h, camera_type=camera_type, - camera_to_worlds=camera_state.c2w.to(torch.float32), + camera_to_worlds=camera_state.c2w.to(torch.float32)[None, ...], times=torch.tensor([0.0], dtype=torch.float32), ) return camera diff --git a/nerfstudio/viewer_beta/viewer.py b/nerfstudio/viewer_beta/viewer.py index 92e609f07..2b6f4e140 100644 --- a/nerfstudio/viewer_beta/viewer.py +++ b/nerfstudio/viewer_beta/viewer.py @@ -291,13 +291,16 @@ class Viewer: self.camera_handles[idx].visible = visible def update_camera_poses(self): + # TODO this fn accounts for like ~5% of total train time # Update the train camera locations based on optimization assert self.camera_handles is not None - idxs = list(self.camera_handles.keys()) if hasattr(self.pipeline.datamanager, "train_camera_optimizer"): camera_optimizer = self.pipeline.datamanager.train_camera_optimizer - else: + elif hasattr(self.pipeline.model, "camera_optimizer"): camera_optimizer = self.pipeline.model.camera_optimizer + else: + return + idxs = list(self.camera_handles.keys()) with torch.no_grad(): assert isinstance(camera_optimizer, CameraOptimizer) c2ws_delta = camera_optimizer(torch.tensor(idxs, device=camera_optimizer.device)).cpu().numpy() @@ -378,7 +381,7 @@ class Viewer: camera = train_dataset.cameras[idx] image_uint8 = (image * 255).detach().type(torch.uint8) image_uint8 = image_uint8.permute(2, 0, 1) - image_uint8 = torchvision.transforms.functional.resize(image_uint8, 100) # type: ignore + image_uint8 = torchvision.transforms.functional.resize(image_uint8, 100, antialias=None) # type: ignore image_uint8 = image_uint8.permute(1, 2, 0) image_uint8 = image_uint8.cpu().numpy() c2w = camera.camera_to_worlds.cpu().numpy() @@ -420,7 +423,6 @@ class Viewer: # this stops training while moving to make the response smoother while time.time() - self.last_move_time < 0.1: time.sleep(0.05) - # self.render_statemachine.action(RenderAction("static", self.camera_state)) if self.trainer is not None and self.trainer.training_state == "training" and self.train_util != 1: if ( EventName.TRAIN_RAYS_PER_SEC.value in GLOBAL_BUFFER["events"] diff --git a/pyproject.toml b/pyproject.toml index 019ef6b6a..fd058f9e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,12 +56,14 @@ dependencies = [ "torchvision>=0.14.1", "torchmetrics[image]>=1.0.1", "typing_extensions>=4.4.0", - "viser==0.1.13", + "viser==0.1.14", "nuscenes-devkit>=1.1.1", "wandb>=0.13.3", "xatlas", "trimesh>=3.20.2", - "timm==0.6.7" + "timm==0.6.7", + "gsplat==0.1.0", + "pytorch-msssim" ] [project.urls] diff --git a/tests/test_train.py b/tests/test_train.py index a88283a59..45d8e348f 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -26,6 +26,7 @@ BLACKLIST = [ "neus", "generfacto", "neus-facto", + "gaussian-splatting", ]