From bb90b94fb025f72180c2281621a76a9f32cbbfbb Mon Sep 17 00:00:00 2001 From: "Jake Austin jake-austin@berkeley.edu" Date: Tue, 12 Sep 2023 03:15:11 +0000 Subject: [PATCH] Basic working, still needs to be profiled and tuned, some preliminary TODOs labeled 'jake-austin' need to be addressed --- nerfstudio/configs/method_configs.py | 95 ++++- .../data/datamanagers/base_datamanager.py | 16 +- .../datamanagers/full_images_datamanager.py | 19 +- nerfstudio/fields/gaussian_pointcloud.py | 4 + nerfstudio/models/base_model.py | 14 +- nerfstudio/models/gaussian_splatting.py | 400 ++++++++++++++++++ nerfstudio/pipelines/base_pipeline.py | 35 +- nerfstudio/utils/gaussian_utils.py | 364 ++++++++++++++++ .../viewer/server/render_state_machine.py | 24 +- .../viewer_beta/render_state_machine.py | 24 +- tests/test_train.py | 1 + 11 files changed, 934 insertions(+), 62 deletions(-) create mode 100644 nerfstudio/utils/gaussian_utils.py diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index a92c042bf..869ad1176 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -22,35 +22,44 @@ from collections import OrderedDict from typing import Dict import tyro -from nerfstudio.data.pixel_samplers import PairPixelSamplerConfig from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig from nerfstudio.configs.base_config import ViewerConfig from nerfstudio.configs.external_methods import get_external_methods - -from nerfstudio.data.datamanagers.random_cameras_datamanager import RandomCamerasDataManagerConfig -from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager, VanillaDataManagerConfig - -from nerfstudio.data.dataparsers.blender_dataparser import BlenderDataParserConfig +from nerfstudio.data.datamanagers.base_datamanager import ( + VanillaDataManager, VanillaDataManagerConfig) +from nerfstudio.data.datamanagers.full_images_datamanager import \ + FullImageDatamanagerConfig +from nerfstudio.data.datamanagers.random_cameras_datamanager import \ + RandomCamerasDataManagerConfig +from nerfstudio.data.dataparsers.blender_dataparser import \ + BlenderDataParserConfig from nerfstudio.data.dataparsers.dnerf_dataparser import DNeRFDataParserConfig -from nerfstudio.data.dataparsers.instant_ngp_dataparser import InstantNGPDataParserConfig -from nerfstudio.data.dataparsers.nerfstudio_dataparser import NerfstudioDataParserConfig -from nerfstudio.data.dataparsers.phototourism_dataparser import PhototourismDataParserConfig -from nerfstudio.data.dataparsers.sdfstudio_dataparser import SDFStudioDataParserConfig -from nerfstudio.data.dataparsers.sitcoms3d_dataparser import Sitcoms3DDataParserConfig +from nerfstudio.data.dataparsers.instant_ngp_dataparser import \ + InstantNGPDataParserConfig +from nerfstudio.data.dataparsers.nerfstudio_dataparser import \ + NerfstudioDataParserConfig +from nerfstudio.data.dataparsers.phototourism_dataparser import \ + PhototourismDataParserConfig +from nerfstudio.data.dataparsers.sdfstudio_dataparser import \ + SDFStudioDataParserConfig +from nerfstudio.data.dataparsers.sitcoms3d_dataparser import \ + Sitcoms3DDataParserConfig from nerfstudio.data.datasets.depth_dataset import DepthDataset from nerfstudio.data.datasets.sdf_dataset import SDFDataset from nerfstudio.data.datasets.semantic_dataset import SemanticDataset -from nerfstudio.engine.optimizers import AdamOptimizerConfig, RAdamOptimizerConfig -from nerfstudio.engine.schedulers import ( - CosineDecaySchedulerConfig, - ExponentialDecaySchedulerConfig, - MultiStepSchedulerConfig, -) +from nerfstudio.data.pixel_samplers import PairPixelSamplerConfig +from nerfstudio.engine.optimizers import (AdamOptimizerConfig, + RAdamOptimizerConfig) +from nerfstudio.engine.schedulers import (CosineDecaySchedulerConfig, + ExponentialDecaySchedulerConfig, + MultiStepSchedulerConfig) from nerfstudio.engine.trainer import TrainerConfig -from nerfstudio.field_components.temporal_distortions import TemporalDistortionKind +from nerfstudio.field_components.temporal_distortions import \ + TemporalDistortionKind from nerfstudio.fields.sdf_field import SDFFieldConfig from nerfstudio.models.depth_nerfacto import DepthNerfactoModelConfig +from nerfstudio.models.gaussian_splatting import GaussianSplattingModelConfig from nerfstudio.models.generfacto import GenerfactoModelConfig from nerfstudio.models.instant_ngp import InstantNGPModelConfig from nerfstudio.models.mipnerf import MipNerfModel @@ -79,6 +88,7 @@ descriptions = { "generfacto": "Generative Text to NeRF model", "neus": "Implementation of NeuS. (slow)", "neus-facto": "Implementation of NeuS-Facto. (slow)", + "gaussian-splatting": "Gaussian Splatting model", } method_configs["nerfacto"] = TrainerConfig( @@ -575,6 +585,55 @@ method_configs["neus-facto"] = TrainerConfig( vis="viewer", ) +method_configs["gaussian-splatting"] = TrainerConfig( + method_name="gaussian-splatting", + steps_per_eval_image=10, + steps_per_eval_batch=10, + steps_per_save=2000, + steps_per_eval_all_images=1000000, # set to a very large model so we don't eval with all images + max_num_iterations=20001, + mixed_precision=False, + pipeline=VanillaPipelineConfig( + datamanager=FullImageDatamanagerConfig( + dataparser=BlenderDataParserConfig(), + ), + model=GaussianSplattingModelConfig(), + ), + optimizers={ + "xyz": { + "optimizer": AdamOptimizerConfig(lr=0.00016*5, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig( + lr_pre_warmup=0.01 * 0.00016*5, + lr_final=0.0000016*5, + warmup_steps = 0, + max_steps=30000, + ), + }, + "f_dc": { + "optimizer": AdamOptimizerConfig(lr=0.0025, eps=1e-15), + "scheduler": None, + }, + "f_rest": { + "optimizer": AdamOptimizerConfig(lr=0.0025, eps=1e-15), + "scheduler": None, + }, + "opacity": { + "optimizer": AdamOptimizerConfig(lr=0.05, eps=1e-15), + "scheduler": None, + }, + "scaling": { + "optimizer": AdamOptimizerConfig(lr=0.001, eps=1e-15), + "scheduler": None, + }, + "rotation": { + "optimizer": AdamOptimizerConfig(lr=0.001, eps=1e-15), + "scheduler": None, + }, + }, + viewer=ViewerConfig(num_rays_per_chunk=1 << 15), + vis="viewer", +) + def merge_methods(methods, method_descriptions, new_methods, new_descriptions, overwrite=True): """Merge new methods and descriptions into existing methods and descriptions. diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index 107212d37..9b901f6be 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -33,7 +33,7 @@ from torch.utils.data.distributed import DistributedSampler from typing_extensions import TypeVar from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig -from nerfstudio.cameras.cameras import CameraType +from nerfstudio.cameras.cameras import Cameras, CameraType from nerfstudio.cameras.rays import RayBundle from nerfstudio.configs.base_config import InstantiateConfig from nerfstudio.configs.dataparser_configs import AnnotatedDataParserUnion @@ -115,7 +115,7 @@ class DataManager(nn.Module): To get data, use the next_train and next_eval functions. This data manager's next_train and next_eval methods will return 2 things: - 1. A Raybundle: This will contain the rays we are sampling, with latents and + 1. A rays: This will contain the rays/camera we are sampling, with latents and conditionals attached (everything needed at inference) 2. A "batch" of auxiliary information: This will contain the mask, the ground truth pixels, etc needed to actually train, score, etc the model @@ -230,7 +230,7 @@ class DataManager(nn.Module): """Sets up the data manager for evaluation""" @abstractmethod - def next_train(self, step: int) -> Tuple[RayBundle, Dict]: + def next_train(self, step: int) -> Tuple[Union[RayBundle, Cameras], Dict]: """Returns the next batch of data from the train data manager. Args: @@ -242,25 +242,25 @@ class DataManager(nn.Module): raise NotImplementedError @abstractmethod - def next_eval(self, step: int) -> Tuple[RayBundle, Dict]: + def next_eval(self, step: int) -> Tuple[Union[RayBundle, Cameras], Dict]: """Returns the next batch of data from the eval data manager. Args: step: the step number of the eval image to retrieve Returns: - A tuple of the ray bundle for the image, and a dictionary of additional batch information + A tuple of the ray/camera for the image, and a dictionary of additional batch information such as the groundtruth image. """ raise NotImplementedError @abstractmethod - def next_eval_image(self, step: int) -> Tuple[int, RayBundle, Dict]: + def next_eval_image(self, step: int) -> Tuple[int, Union[RayBundle, Cameras], Dict]: """Retrieve the next eval image. Args: step: the step number of the eval image to retrieve Returns: - A tuple of the step number, the ray bundle for the image, and a dictionary of + A tuple of the step number, the ray/camera for the image, and a dictionary of additional batch information such as the groundtruth image. """ raise NotImplementedError @@ -297,7 +297,7 @@ class DataManager(nn.Module): @dataclass class VanillaDataManagerConfig(DataManagerConfig): - """A basic data manager""" + """A basic data manager for a ray-based model""" _target: Type = field(default_factory=lambda: VanillaDataManager) """Target class to instantiate.""" diff --git a/nerfstudio/data/datamanagers/full_images_datamanager.py b/nerfstudio/data/datamanagers/full_images_datamanager.py index a7ac96471..27a49f8c0 100644 --- a/nerfstudio/data/datamanagers/full_images_datamanager.py +++ b/nerfstudio/data/datamanagers/full_images_datamanager.py @@ -21,6 +21,7 @@ paradigm from __future__ import annotations +import random import sys from abc import abstractmethod from collections import defaultdict @@ -29,7 +30,7 @@ from functools import cached_property from pathlib import Path from typing import (Any, Callable, Dict, ForwardRef, Generic, List, Literal, Optional, Tuple, Type, Union, cast, get_args, get_origin) -import random + import torch from rich.progress import Console from torch import Tensor, nn @@ -87,6 +88,9 @@ class FullImageDatamanager(DataManager, Generic[TDataset]): A datamanager that outputs full images and cameras instead of raybundles. This makes the datamanager far more lightweight since we dont have to do ray generation for things like gaussian splatting that dont require it. + + TODO (jake-austin): Figure out why the dataparser doesnt show up as a subcommand, preventing us + from specifying nerfstudio-data instead of blender-data """ config: FullImageDatamanagerConfig @@ -191,6 +195,10 @@ class FullImageDatamanager(DataManager, Generic[TDataset]): """ return {} + def get_train_rays_per_batch(self): + # TODO (jake-austin): fix this to be the resolution of the last image rendered + return 800*800 + def next_train(self, step: int) -> Tuple[Cameras, Dict]: """Returns the next training batch @@ -200,8 +208,9 @@ class FullImageDatamanager(DataManager, Generic[TDataset]): if len(self.train_unseen_cameras) == 0: self.train_unseen_cameras = [i for i in range(len(self.train_dataset))] data = self.train_dataset.get_data(image_idx) + data["image"] = data["image"].to(self.device) assert len(self.train_dataset.cameras.shape) == 1, "Assumes single batch dimension" - camera = self.train_dataset.cameras[image_idx] + camera = self.train_dataset.cameras[image_idx].to(self.device) return camera, data def next_eval(self, step: int) -> Tuple[Cameras, Dict]: @@ -213,8 +222,9 @@ class FullImageDatamanager(DataManager, Generic[TDataset]): if len(self.eval_unseen_cameras) == 0: self.eval_unseen_cameras = [i for i in range(len(self.eval_dataset))] data = self.eval_dataset.get_data(image_idx) + data["image"] = data["image"].to(self.device) assert len(self.eval_dataset.cameras.shape) == 1, "Assumes single batch dimension" - camera = self.eval_dataset.cameras[image_idx] + camera = self.eval_dataset.cameras[image_idx].to(self.device) return camera, data def next_eval_image(self, step: int) -> Tuple[int, Cameras, Dict]: @@ -228,6 +238,7 @@ class FullImageDatamanager(DataManager, Generic[TDataset]): if len(self.eval_unseen_cameras) == 0: self.eval_unseen_cameras = [i for i in range(len(self.eval_dataset))] data = self.eval_dataset.get_data(image_idx) + data["image"] = data["image"].to(self.device) assert len(self.eval_dataset.cameras.shape) == 1, "Assumes single batch dimension" - camera = self.eval_dataset.cameras[image_idx] + camera = self.eval_dataset.cameras[image_idx].to(self.device) return image_idx, camera, data diff --git a/nerfstudio/fields/gaussian_pointcloud.py b/nerfstudio/fields/gaussian_pointcloud.py index 29e44f9d2..d715d7485 100644 --- a/nerfstudio/fields/gaussian_pointcloud.py +++ b/nerfstudio/fields/gaussian_pointcloud.py @@ -26,6 +26,10 @@ from utils.system_utils import mkdir_p class GaussianPointcloud: + # TODO (jake-austin): Make sure these methods / properties are clean af, and maybe make a superclass + # for pointclouds in general since people will likely make new kinds under this gaussian pointcloud + # paradigm + def setup_functions(self): def build_covariance_from_scaling_rotation(scaling, scaling_modifier, rotation): L = build_scaling_rotation(scaling_modifier * scaling, rotation) diff --git a/nerfstudio/models/base_model.py b/nerfstudio/models/base_model.py index ebda710b3..aa5562895 100644 --- a/nerfstudio/models/base_model.py +++ b/nerfstudio/models/base_model.py @@ -13,7 +13,7 @@ # limitations under the License. """ -Base Model implementation which takes in RayBundles +Base Model implementation which takes in RayBundles or Cameras """ from __future__ import annotations @@ -27,11 +27,13 @@ import torch from torch import nn from torch.nn import Parameter +from nerfstudio.cameras.cameras import Cameras from nerfstudio.cameras.rays import RayBundle from nerfstudio.configs.base_config import InstantiateConfig from nerfstudio.configs.config_utils import to_immutable_dict from nerfstudio.data.scene_box import SceneBox -from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes +from nerfstudio.engine.callbacks import (TrainingCallback, + TrainingCallbackAttributes) from nerfstudio.model_components.scene_colliders import NearFarCollider @@ -117,7 +119,7 @@ class Model(nn.Module): """ @abstractmethod - def get_outputs(self, ray_bundle: RayBundle) -> Dict[str, Union[torch.Tensor, List]]: + def get_outputs(self, ray_bundle: Union[RayBundle, Cameras]) -> Dict[str, Union[torch.Tensor, List]]: """Takes in a Ray Bundle and returns a dictionary of outputs. Args: @@ -128,7 +130,7 @@ class Model(nn.Module): Outputs of model. (ie. rendered colors) """ - def forward(self, ray_bundle: RayBundle) -> Dict[str, Union[torch.Tensor, List]]: + def forward(self, ray_bundle: Union[RayBundle, Cameras]) -> Dict[str, Union[torch.Tensor, List]]: """Run forward starting with a ray bundle. This outputs different things depending on the configuration of the model and whether or not the batch is provided (whether or not we are training basically) @@ -162,9 +164,9 @@ class Model(nn.Module): """ @torch.no_grad() - def get_outputs_for_camera_ray_bundle(self, camera_ray_bundle: RayBundle) -> Dict[str, torch.Tensor]: + def get_outputs_for_camera_ray_bundle(self, camera_ray_bundle: Union[RayBundle, Cameras]) -> Dict[str, torch.Tensor]: """Takes in camera parameters and computes the output of the model. - + TODO (jake-austin): Make 2 cases for when this is a ray or camera based model Args: camera_ray_bundle: ray bundle to calculate outputs over """ diff --git a/nerfstudio/models/gaussian_splatting.py b/nerfstudio/models/gaussian_splatting.py index e69de29bb..84a8bea91 100644 --- a/nerfstudio/models/gaussian_splatting.py +++ b/nerfstudio/models/gaussian_splatting.py @@ -0,0 +1,400 @@ +# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +NeRF implementation that combines many recent advancements. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Literal, Optional, Tuple, Type, Union + +import diff_rast +import matplotlib.pyplot as plt +import numpy as np +import torch +import torch.nn as nn +from torch.nn import Parameter +from torchmetrics.functional import structural_similarity_index_measure +from torchmetrics.image import PeakSignalNoiseRatio +from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity + +from nerfstudio.cameras.cameras import Cameras, CameraType +from nerfstudio.cameras.rays import RayBundle, RaySamples +from nerfstudio.engine.callbacks import (TrainingCallback, + TrainingCallbackAttributes, + TrainingCallbackLocation) +from nerfstudio.field_components.field_heads import FieldHeadNames +from nerfstudio.field_components.spatial_distortions import SceneContraction +from nerfstudio.fields.density_fields import HashMLPDensityField +from nerfstudio.fields.nerfacto_field import NerfactoField +from nerfstudio.model_components.losses import ( + MSELoss, distortion_loss, interlevel_loss, orientation_loss, + pred_normal_loss, scale_gradients_by_distance_squared) +from nerfstudio.model_components.ray_samplers import (ProposalNetworkSampler, + UniformSampler) +from nerfstudio.model_components.renderers import (AccumulationRenderer, + DepthRenderer, + NormalsRenderer, + RGBRenderer) +from nerfstudio.model_components.scene_colliders import NearFarCollider +from nerfstudio.model_components.shaders import NormalsShader +from nerfstudio.models.base_model import Model, ModelConfig +from nerfstudio.utils import colormaps, gaussian_utils +from nerfstudio.utils import poses as pose_utils + + +@dataclass +class GaussianSplattingModelConfig(ModelConfig): + """Gaussian Splatting Model Config""" + _target: Type = field(default_factory=lambda: GaussianSplattingModel) + initial_ptc_size = 100000 + max_ptc_size = 100000 + render_accumulation = False + use_aabb = True + initial_gaussian_scales = -3 + prune_density_after = 500 + prune_density_every = 100 + prune_density_until = 15000 + one_up_sh_every = 1000 + lambda_ssim = 0.2 + +class GaussianSplattingModel(Model): + """Gaussian Splatting model + + Args: + config: Gaussian Splatting configuration to instantiate model + """ + + config: GaussianSplattingModelConfig + + def populate_modules(self): + # TODO (jake-austin): clean this up, this is transplanted code across all the implementation functions + self.active_sh_degree = 0 + self.max_sh_degree = 3 + + self._xyz = torch.empty(0) + self._features_dc = torch.empty(0) + self._features_rest = torch.empty(0) + self._scaling = torch.empty(0) + self._rotation = torch.empty(0) + self._opacity = torch.empty(0) + self.max_radii2D = torch.empty(0) + self.xyz_gradient_accum = torch.empty(0) + self.denom = torch.empty(0) + self.optimizer = None + self.percent_dense = 0 + self.spatial_lr_scale = 0 + self.max_points = self.config.max_ptc_size + + + def build_covariance_from_scaling_rotation(scaling, scaling_modifier, rotation): + L = gaussian_utils.build_scaling_rotation(scaling_modifier * scaling, rotation) + actual_covariance = L @ L.transpose(1, 2) + symm = gaussian_utils.strip_symmetric(actual_covariance) + return symm + + self.scaling_activation = torch.exp + self.scaling_inverse_activation = torch.log + + self.covariance_activation = build_covariance_from_scaling_rotation + + self.opacity_activation = torch.sigmoid + self.inverse_opacity_activation = gaussian_utils.inverse_sigmoid + + self.rotation_activation = torch.nn.functional.normalize + + num_pts = self.config.initial_ptc_size + print(f"Generating random point cloud ({num_pts})...") + + # We create random points inside the bounds of the synthetic Blender scenes + xyz = np.random.random((num_pts, 3)) * 2.6 - 1.3 + shs = np.random.random((num_pts, 3)) / 255.0 + # pcd = BasicPointCloud(points=xyz, colors=SH2RGB(shs), normals=np.zeros((num_pts, 3))) + + self.spatial_lr_scale = 5 + + fused_point_cloud = torch.tensor(np.asarray(xyz)).float().cuda() + fused_color = gaussian_utils.RGB2SH(torch.tensor(np.asarray(gaussian_utils.SH2RGB(shs))).float().cuda()) + features = torch.zeros((fused_color.shape[0], 3, (self.max_sh_degree + 1) ** 2)).float().cuda() + features[:, :3, 0 ] = fused_color + features[:, 3:, 1:] = 0.0 + + # dist2 = torch.clamp_min(distCUDA2(torch.from_numpy(np.asarray(pcd.points)).float().cuda()), 0.0000001) + scales = torch.ones((fused_point_cloud.shape[0], 3), device="cuda") * self.config.initial_gaussian_scales + rots = torch.zeros((fused_point_cloud.shape[0], 4), device="cuda") + rots[:, 0] = 1 + + opacities = gaussian_utils.inverse_sigmoid(0.1 * torch.ones((fused_point_cloud.shape[0], 1), dtype=torch.float, device="cuda")) + + self._xyz = nn.Parameter(fused_point_cloud.requires_grad_(True)) + self._features_dc = nn.Parameter(features[:,:,0:1].transpose(1, 2).contiguous().requires_grad_(True)) + self._features_rest = nn.Parameter(features[:,:,1:].transpose(1, 2).contiguous().requires_grad_(True)) + self._scaling = nn.Parameter(scales.requires_grad_(True)) + self._rotation = nn.Parameter(rots.requires_grad_(True)) + self._opacity = nn.Parameter(opacities.requires_grad_(True)) + self.max_radii2D = torch.zeros((self._xyz.shape[0]), device="cuda") + + self.percent_dense = 0.01 + self.xyz_gradient_accum = torch.zeros((self._xyz.shape[0], 1), device="cuda") + self.denom = torch.zeros((self._xyz.shape[0], 1), device="cuda") + + # l = [ + # {'params': [self._xyz], 'lr': training_args.position_lr_init*self.spatial_lr_scale, "name": "xyz"}, + # {'params': [self._features_dc], 'lr': training_args.feature_lr, "name": "f_dc"}, + # {'params': [self._features_rest], 'lr': training_args.feature_lr / 20.0, "name": "f_rest"}, + # {'params': [self._opacity], 'lr': training_args.opacity_lr, "name": "opacity"}, + # {'params': [self._scaling], 'lr': training_args.scaling_lr*self.spatial_lr_scale, "name": "scaling"}, + # {'params': [self._rotation], 'lr': training_args.rotation_lr, "name": "rotation"} + # ] + + # self.optimizer = torch.optim.Adam(l, lr=0.0, eps=1e-15) + # self.xyz_scheduler_args = gaussian_utils.get_expon_lr_func(lr_init=training_args.position_lr_init*self.spatial_lr_scale, + # lr_final=training_args.position_lr_final*self.spatial_lr_scale, + # lr_delay_mult=training_args.position_lr_delay_mult, + # max_steps=training_args.position_lr_max_steps) + + self.device_indicator_param = nn.Parameter(torch.empty(0)) + + # metrics + self.psnr = PeakSignalNoiseRatio(data_range=1.0) + self.ssim = structural_similarity_index_measure + self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True) + + @property + def get_scaling(self): + return self.scaling_activation(self._scaling) + + @property + def get_rotation(self): + return self.rotation_activation(self._rotation) + + @property + def get_xyz(self): + return self._xyz + + @property + def get_features(self): + features_dc = self._features_dc + features_rest = self._features_rest + return torch.cat((features_dc, features_rest), dim=1) + + @property + def get_opacity(self): + return self.opacity_activation(self._opacity) + + def get_covariance(self, scaling_modifier = 1): + return self.covariance_activation(self.get_scaling, scaling_modifier, self._rotation) + + @property + def device(self): + """Returns the device that the model is on.""" + return self.device_indicator_param.device + + def get_training_callbacks( + self, training_callback_attributes: TrainingCallbackAttributes + ) -> List[TrainingCallback]: + + # TODO (jake-austin): remove and implement this + return [] + + callbacks = [] + + def oneupSHdegree(self): + if self.active_sh_degree < self.max_sh_degree: + self.active_sh_degree += 1 + def wrapper_pruning_densification(step): + if step <= self.config.prune_density_after and step > self.config.prune_density_until: + return + self.pruning_densification( + training_callback_attributes.optimizers, + training_callback_attributes.grad_scaler + ) + def wrapper_reset_opacity(step): + if step > self.config.prune_density_until: + return + self.reset_opacity( + training_callback_attributes.optimizers, + training_callback_attributes.grad_scaler + ) + + callbacks.append( + TrainingCallback( + where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION], + update_every_num_iters=self.config.one_up_sh_every, + func=oneupSHdegree, + ) + ) + callbacks.append( + TrainingCallback( + where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION], + update_every_num_iters=self.config.prune_density_every, + func=wrapper_pruning_densification, + ) + ) + + return callbacks + + def pruning_densification(self, optimizers, grad_scaler): + assert False + + def reset_opacity(self, optimizers, grad_scaler): + opacities_new = gaussian_utils.inverse_sigmoid(torch.min(self.get_opacity, torch.ones_like(self.get_opacity)*0.01)) + optimizable_tensors = self.update_optimizer_tensors({"opacity": opacities_new}, "replace") + self._opacity = optimizable_tensors["opacity"] + + def update_optimizer_tensors(self, data: dict, operation: Literal["cat", "prune", "replace"]): + optimizable_tensors = {} + for group in self.optimizer.param_groups: + if group["name"] in data: + if operation == "cat": + # TODO (jake-austin): implement this, consider splitting this into 3 functions? + assert False + elif operation == "prune": + assert False + elif operation == "replace": + assert False + else: + raise ValueError(f"Unknown operation {operation}") + + def get_param_groups(self) -> Dict[str, List[Parameter]]: + """Obtain the parameter groups for the optimizers + + Returns: + Mapping of different parameter groups + """ + return { + "xyz": [self._xyz], + "f_dc": [self._features_dc], + "f_rest": [self._features_rest], + "opacity": [self._opacity], + "scaling": [self._scaling], + "rotation": [self._rotation], + } + + def get_outputs(self, ray_bundle: Cameras) -> Dict[str, Union[torch.Tensor, List]]: + """Takes in a Ray Bundle and returns a dictionary of outputs. + + TODO (jake-austin): use the new homebrew nerfstudio gaussian rasterization code instead + + Args: + ray_bundle: Input bundle of rays. This raybundle should have all the + needed information to compute the outputs. + + Returns: + Outputs of model. (ie. rendered colors) + """ + ray_bundle = ray_bundle.reshape(()) + assert ray_bundle.shape == () + R = ray_bundle.camera_to_worlds[..., :3, :3].squeeze() + R[:,0] = -R[:,0] + T = (R.T @ ray_bundle.camera_to_worlds[..., :3, 3:4]).squeeze() + fovx = gaussian_utils.focal2fov(ray_bundle.fx, ray_bundle.width) + fovy = gaussian_utils.focal2fov(ray_bundle.fy, ray_bundle.height) + + world_view_transform = gaussian_utils.getWorld2View2(R, T, torch.tensor([0.0, 0.0, 0.0]).to(self.device), 1.0).transpose(0, 1).to(self.device) + projection_matrix = gaussian_utils.getProjectionMatrix(znear=.01, zfar=100, fovX=fovx, fovY=fovy).transpose(0,1).to(self.device) + full_projection_matrix = (world_view_transform.unsqueeze(0).bmm(projection_matrix.unsqueeze(0))).squeeze(0) + + camera_center = world_view_transform.inverse()[3, :3] + data = { + "xyz": self.get_xyz, + "features": self.get_features, + "opacity": self.get_opacity, + "scaling": self.get_scaling, + "rotation": self.get_rotation, + "active_sh_degree": self.active_sh_degree, + "max_sh_degree": self.max_sh_degree, + "FoVx": fovx, + "FoVy": fovy, + "image_height": ray_bundle.width, + "image_width": ray_bundle.width, + "world_view_transform": world_view_transform, + "full_proj_transform": full_projection_matrix, + "camera_center": camera_center, + } + + image = gaussian_utils.render_from_dict( + data, + bg_color=torch.ones(3).to(self.device), + )["render"].permute(1, 2, 0) #HWC + + return {"rgb": image} + + def get_metrics_dict(self, outputs, batch) -> Dict[str, torch.Tensor]: + """Compute and returns metrics. + + Args: + outputs: the output to compute loss dict to + batch: ground truth batch corresponding to outputs + """ + + return {} + + def get_loss_dict(self, outputs, batch, metrics_dict=None) -> Dict[str, torch.Tensor]: + """Computes and returns the losses dict. + + Args: + outputs: the output to compute loss dict to + batch: ground truth batch corresponding to outputs + metrics_dict: dictionary of metrics, some of which we can use for loss + """ + Ll1 = gaussian_utils.l1_loss(batch['image'], outputs['rgb']) + loss = (1.0 - self.config.lambda_ssim) * Ll1 + self.config.lambda_ssim * (1.0 - gaussian_utils.ssim(batch['image'], outputs['rgb'])) + return {"main_loss": loss} + + @torch.no_grad() + def get_outputs_for_camera_ray_bundle(self, camera_ray_bundle: Cameras) -> Dict[str, torch.Tensor]: + """Takes in camera parameters and computes the output of the model. + + Args: + camera_ray_bundle: ray bundle to calculate outputs over + """ + return self.get_outputs(camera_ray_bundle) + + def get_image_metrics_and_images( + self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor] + ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: + """Writes the test image outputs. + + Args: + image_idx: Index of the image. + step: Current step. + batch: Batch of data. + outputs: Outputs of the model. + + Returns: + A dictionary of metrics. + """ + gt_rgb = batch["image"].to(self.device) + predicted_rgb = outputs["rgb"] # Blended with background (black if random background) + + combined_rgb = torch.cat([gt_rgb, predicted_rgb], dim=1) + + # Switch images from [H, W, C] to [1, C, H, W] for metrics computations + gt_rgb = torch.moveaxis(gt_rgb, -1, 0)[None, ...] + predicted_rgb = torch.moveaxis(predicted_rgb, -1, 0)[None, ...] + + psnr = self.psnr(gt_rgb, predicted_rgb) + ssim = self.ssim(gt_rgb, predicted_rgb) + lpips = self.lpips(gt_rgb, predicted_rgb) + + # all of these metrics will be logged as scalars + metrics_dict = {"psnr": float(psnr.item()), "ssim": float(ssim)} # type: ignore + metrics_dict["lpips"] = float(lpips) + + images_dict = {"img": combined_rgb} + + return metrics_dict, images_dict diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py index 5e202b06c..388d3f9f9 100644 --- a/nerfstudio/pipelines/base_pipeline.py +++ b/nerfstudio/pipelines/base_pipeline.py @@ -22,30 +22,26 @@ from abc import abstractmethod from dataclasses import dataclass, field from pathlib import Path from time import time -from typing import Any, Dict, List, Literal, Mapping, Optional, Tuple, Type, Union, cast +from typing import (Any, Dict, List, Literal, Mapping, Optional, Tuple, Type, + Union, cast) import torch import torch.distributed as dist from PIL import Image -from rich.progress import ( - BarColumn, - MofNCompleteColumn, - Progress, - TextColumn, - TimeElapsedColumn, -) +from rich.progress import (BarColumn, MofNCompleteColumn, Progress, TextColumn, + TimeElapsedColumn) from torch import nn +from torch.cuda.amp.grad_scaler import GradScaler from torch.nn import Parameter from torch.nn.parallel import DistributedDataParallel as DDP -from torch.cuda.amp.grad_scaler import GradScaler +from nerfstudio.cameras.cameras import Cameras from nerfstudio.configs import base_config as cfg -from nerfstudio.data.datamanagers.base_datamanager import ( - DataManager, - DataManagerConfig, - VanillaDataManager, -) -from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes +from nerfstudio.data.datamanagers.base_datamanager import (DataManager, + DataManagerConfig, + VanillaDataManager) +from nerfstudio.engine.callbacks import (TrainingCallback, + TrainingCallbackAttributes) from nerfstudio.models.base_model import Model, ModelConfig from nerfstudio.utils import profiler @@ -352,7 +348,12 @@ class VanillaPipeline(Pipeline): assert "image_idx" not in metrics_dict metrics_dict["image_idx"] = image_idx assert "num_rays" not in metrics_dict - metrics_dict["num_rays"] = len(camera_ray_bundle) + if type(camera_ray_bundle) is Cameras: + metrics_dict["num_rays"] = ( + camera_ray_bundle.height * camera_ray_bundle.width * camera_ray_bundle.size + ).item() + else: + metrics_dict["num_rays"] = len(camera_ray_bundle) self.train() return metrics_dict, images_dict @@ -372,6 +373,8 @@ class VanillaPipeline(Pipeline): """ self.eval() metrics_dict_list = [] + # TODO (jake-austin): Can we remove this assert statement? Are all the needed functions + # abstract functions in the upper-most DataManager class? assert isinstance(self.datamanager, VanillaDataManager) num_images = len(self.datamanager.fixed_indices_eval_dataloader) with Progress( diff --git a/nerfstudio/utils/gaussian_utils.py b/nerfstudio/utils/gaussian_utils.py new file mode 100644 index 000000000..74d65cec3 --- /dev/null +++ b/nerfstudio/utils/gaussian_utils.py @@ -0,0 +1,364 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# + +import math +import random +import sys +from datetime import datetime +from typing import NamedTuple + +import numpy as np +import torch +from diff_gaussian_rasterization import (GaussianRasterizationSettings, + GaussianRasterizer) +from torch import nn + +# TODO (jake-austin): remove this all outright as we move to our own homebrew gaussian splatting based rasterizer + +def inverse_sigmoid(x): + return torch.log(x/(1-x)) + +def PILtoTorch(pil_image, resolution): + resized_image_PIL = pil_image.resize(resolution) + resized_image = torch.from_numpy(np.array(resized_image_PIL)) / 255.0 + if len(resized_image.shape) == 3: + return resized_image.permute(2, 0, 1) + else: + return resized_image.unsqueeze(dim=-1).permute(2, 0, 1) + +def strip_lowerdiag(L): + uncertainty = torch.zeros((L.shape[0], 6), dtype=torch.float, device="cuda") + + uncertainty[:, 0] = L[:, 0, 0] + uncertainty[:, 1] = L[:, 0, 1] + uncertainty[:, 2] = L[:, 0, 2] + uncertainty[:, 3] = L[:, 1, 1] + uncertainty[:, 4] = L[:, 1, 2] + uncertainty[:, 5] = L[:, 2, 2] + return uncertainty + +def strip_symmetric(sym): + return strip_lowerdiag(sym) + +def build_rotation(r): + norm = torch.sqrt(r[:,0]*r[:,0] + r[:,1]*r[:,1] + r[:,2]*r[:,2] + r[:,3]*r[:,3]) + + q = r / norm[:, None] + + R = torch.zeros((q.size(0), 3, 3), device='cuda') + + r = q[:, 0] + x = q[:, 1] + y = q[:, 2] + z = q[:, 3] + + R[:, 0, 0] = 1 - 2 * (y*y + z*z) + R[:, 0, 1] = 2 * (x*y - r*z) + R[:, 0, 2] = 2 * (x*z + r*y) + R[:, 1, 0] = 2 * (x*y + r*z) + R[:, 1, 1] = 1 - 2 * (x*x + z*z) + R[:, 1, 2] = 2 * (y*z - r*x) + R[:, 2, 0] = 2 * (x*z - r*y) + R[:, 2, 1] = 2 * (y*z + r*x) + R[:, 2, 2] = 1 - 2 * (x*x + y*y) + return R + +def build_scaling_rotation(s, r): + L = torch.zeros((s.shape[0], 3, 3), dtype=torch.float, device="cuda") + R = build_rotation(r) + + L[:,0,0] = s[:,0] + L[:,1,1] = s[:,1] + L[:,2,2] = s[:,2] + + L = R @ L + return L + +def safe_state(silent): + old_f = sys.stdout + class F: + def __init__(self, silent): + self.silent = silent + + def write(self, x): + if not self.silent: + if x.endswith("\n"): + old_f.write(x.replace("\n", " [{}]\n".format(str(datetime.now().strftime("%d/%m %H:%M:%S"))))) + else: + old_f.write(x) + + def flush(self): + old_f.flush() + + sys.stdout = F(silent) + + random.seed(0) + np.random.seed(0) + torch.manual_seed(0) + torch.cuda.set_device(torch.device("cuda:0")) + +def render_from_dict(data : dict, bg_color : torch.Tensor, scaling_modifier = 1.0, override_color = None): + """ + Render the scene. + + Background tensor (bg_color) must be on GPU! + """ + for k in ['active_sh_degree', 'max_sh_degree', 'xyz', 'scaling', 'rotation', 'features', 'opacity', 'FoVx', 'FoVy', 'image_height', 'image_width', 'world_view_transform', 'full_proj_transform', 'camera_center']: + assert k in data, f'Required key {k} not in {data.keys()}' + + # Create zero tensor. We will use it to make pytorch return gradients of the 2D (screen-space) means + screenspace_points = torch.zeros_like(data['xyz'], dtype=data['xyz'].dtype, requires_grad=True, device="cuda") + 0 + try: + screenspace_points.retain_grad() + except: + pass + + # Set up rasterization configuration + tanfovx = torch.tan(data['FoVx'] * 0.5).item() + tanfovy = torch.tan(data['FoVy'] * 0.5).item() + + raster_settings = GaussianRasterizationSettings( + image_height=int(data['image_height']), + image_width=int(data['image_width']), + tanfovx=tanfovx, + tanfovy=tanfovy, + bg=bg_color, + scale_modifier=scaling_modifier, + viewmatrix=data['world_view_transform'], + projmatrix=data['full_proj_transform'], + sh_degree=data['active_sh_degree'], + campos=data['camera_center'], + prefiltered=False + ) + + rasterizer = GaussianRasterizer(raster_settings=raster_settings) + + means3D = data['xyz'] + means2D = screenspace_points + opacity = data['opacity'] + + # If precomputed 3d covariance is provided, use it. If not, then it will be computed from + # scaling / rotation by the rasterizer. + scales = None + rotations = None + cov3D_precomp = None + # if pipe.compute_cov3D_python: + # cov3D_precomp = pc.get_covariance(scaling_modifier) + # else: + scales = data['scaling'] + rotations = data['rotation'] + + # If precomputed colors are provided, use them. Otherwise, if it is desired to precompute colors + # from SHs in Python, do it. If not, then SH -> RGB conversion will be done by rasterizer. + shs = None + colors_precomp = None + if colors_precomp is None: + shs = data['features'] + else: + + colors_precomp = override_color + + # Rasterize visible Gaussians to image, obtain their radii (on screen). + rendered_image, radii = rasterizer( + means3D = means3D, + means2D = means2D, + shs = shs, + colors_precomp = colors_precomp, + opacities = opacity, + scales = scales, + rotations = rotations, + cov3D_precomp = cov3D_precomp) + + # Those Gaussians that were frustum culled or had a radius of 0 were not visible. + # They will be excluded from value updates used in the splitting criteria. + return {"render": rendered_image, + "viewspace_points": screenspace_points, + "visibility_filter" : None, #radii > 0, + "radii": radii} + + +def render_accumulation_from_dict(viewpoint_camera, pc : dict, bg_color : torch.Tensor, scaling_modifier = 1.0): + """ + TODO (jake-austin): double check that this is actually equivalent to getting the accumulation + """ + override_color = torch.ones_like(pc['xyz'], dtype=pc['xyz'].dtype, requires_grad=True, device="cuda") + 0 + return render_from_dict( + viewpoint_camera, + pc, + bg_color, + scaling_modifier, + override_color=override_color + ) + + +def inverse_sigmoid(x): + return torch.log(x/(1-x)) + +C0 = 0.28209479177387814 + +def RGB2SH(rgb): + return (rgb - 0.5) / C0 + +def SH2RGB(sh): + return sh * C0 + 0.5 + + +def getWorld2View(R, t): + Rt = torch.zeros((4, 4)) + Rt[:3, :3] = R.transpose() + Rt[:3, 3] = t + Rt[3, 3] = 1.0 + return torch.float32(Rt) + +def getWorld2View2(R, t, translate=torch.tensor([.0, .0, .0]), scale=1.0): + Rt = torch.zeros((4, 4)).to(R.device) + Rt[:3, :3] = R.transpose(-1,-2) + Rt[:3, 3] = t + Rt[3, 3] = 1.0 + + C2W = torch.linalg.inv(Rt) + cam_center = C2W[:3, 3] + + cam_center = (cam_center + translate.to(R.device)) * scale + C2W[:3, 3] = cam_center + Rt = torch.linalg.inv(C2W) + return Rt + +def getProjectionMatrix(znear, zfar, fovX, fovY): + tanHalfFovY = torch.tan((fovY / 2)) + tanHalfFovX = torch.tan((fovX / 2)) + + top = tanHalfFovY * znear + bottom = -top + right = tanHalfFovX * znear + left = -right + + P = torch.zeros(4, 4).to(fovX.device) + + z_sign = 1.0 + + P[0, 0] = 2.0 * znear / (right - left) + P[1, 1] = 2.0 * znear / (top - bottom) + P[0, 2] = (right + left) / (right - left) + P[1, 2] = (top + bottom) / (top - bottom) + P[3, 2] = z_sign + P[2, 2] = z_sign * zfar / (zfar - znear) + P[2, 3] = -(zfar * znear) / (zfar - znear) + return P + +def fov2focal(fov, pixels): + return pixels / (2 * torch.tan(fov / 2)) + +def focal2fov(focal, pixels): + return 2*torch.atan(pixels/(2*focal)) + + +class Camera(nn.Module): + def __init__(self, R, T, FoVx, FoVy, height, width, data_device = "cuda"): + super(Camera, self).__init__() + + self.R = R + self.T = T + self.FoVx = FoVx + self.FoVy = FoVy + + try: + self.data_device = torch.device(data_device) + except Exception as e: + print(e) + print(f"[Warning] Custom device {data_device} failed, fallback to default cuda device" ) + self.data_device = torch.device("cuda") + + self.image_width = width + self.image_height = height + + self.zfar = 100.0 + self.znear = 0.01 + + self.trans =torch.tensor([0.0, 0.0, 0.0]).to(self.data_device) + self.scale = 1.0 + + self.world_view_transform = getWorld2View2(R, T, self.trans, self.scale).transpose(0, 1).to(self.data_device) + self.projection_matrix = getProjectionMatrix(znear=self.znear, zfar=self.zfar, fovX=self.FoVx, fovY=self.FoVy).transpose(0,1).to(self.data_device) + self.full_proj_transform = (self.world_view_transform.unsqueeze(0).bmm(self.projection_matrix.unsqueeze(0))).squeeze(0) + self.camera_center = self.world_view_transform.inverse()[3, :3] + +class MiniCam: + def __init__(self, width, height, fovy, fovx, znear, zfar, world_view_transform, full_proj_transform): + self.image_width = width + self.image_height = height + self.FoVy = fovy + self.FoVx = fovx + self.znear = znear + self.zfar = zfar + self.world_view_transform = world_view_transform + self.full_proj_transform = full_proj_transform + view_inv = torch.inverse(self.world_view_transform) + self.camera_center = view_inv[3][:3] + + + + +from math import exp + +import torch +import torch.nn.functional as F +from torch.autograd import Variable + + +def l1_loss(network_output, gt): + return torch.abs((network_output - gt)).mean() + +def l2_loss(network_output, gt): + return ((network_output - gt) ** 2).mean() + +def gaussian(window_size, sigma): + gauss = torch.Tensor([exp(-(x - window_size // 2) ** 2 / float(2 * sigma ** 2)) for x in range(window_size)]) + return gauss / gauss.sum() + +def create_window(window_size, channel): + _1D_window = gaussian(window_size, 1.5).unsqueeze(1) + _2D_window = _1D_window.mm(_1D_window.t()).float().unsqueeze(0).unsqueeze(0) + window = Variable(_2D_window.expand(channel, 1, window_size, window_size).contiguous()) + return window + +def ssim(img1, img2, window_size=11, size_average=True): + channel = img1.size(-3) + window = create_window(window_size, channel) + + if img1.is_cuda: + window = window.cuda(img1.get_device()) + window = window.type_as(img1) + + return _ssim(img1, img2, window, window_size, channel, size_average) + +def _ssim(img1, img2, window, window_size, channel, size_average=True): + mu1 = F.conv2d(img1, window, padding=window_size // 2, groups=channel) + mu2 = F.conv2d(img2, window, padding=window_size // 2, groups=channel) + + mu1_sq = mu1.pow(2) + mu2_sq = mu2.pow(2) + mu1_mu2 = mu1 * mu2 + + sigma1_sq = F.conv2d(img1 * img1, window, padding=window_size // 2, groups=channel) - mu1_sq + sigma2_sq = F.conv2d(img2 * img2, window, padding=window_size // 2, groups=channel) - mu2_sq + sigma12 = F.conv2d(img1 * img2, window, padding=window_size // 2, groups=channel) - mu1_mu2 + + C1 = 0.01 ** 2 + C2 = 0.03 ** 2 + + ssim_map = ((2 * mu1_mu2 + C1) * (2 * sigma12 + C2)) / ((mu1_sq + mu2_sq + C1) * (sigma1_sq + sigma2_sq + C2)) + + if size_average: + return ssim_map.mean() + else: + return ssim_map.mean(1).mean(1).mean(1) + diff --git a/nerfstudio/viewer/server/render_state_machine.py b/nerfstudio/viewer/server/render_state_machine.py index 390390d48..2a01b1a47 100644 --- a/nerfstudio/viewer/server/render_state_machine.py +++ b/nerfstudio/viewer/server/render_state_machine.py @@ -23,7 +23,8 @@ from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Tuple, get_args import torch from nerfstudio.cameras.cameras import Cameras -from nerfstudio.model_components.renderers import background_color_override_context +from nerfstudio.model_components.renderers import \ + background_color_override_context from nerfstudio.utils import colormaps, writer from nerfstudio.utils.writer import GLOBAL_BUFFER, EventName, TimeWriter from nerfstudio.viewer.server import viewer_utils @@ -127,7 +128,11 @@ class RenderStateMachine(threading.Thread): assert camera is not None, "render called before viewer connected" with self.viewer.train_lock if self.viewer.train_lock is not None else contextlib.nullcontext(): - camera_ray_bundle = camera.generate_rays(camera_indices=0, aabb_box=self.viewer.get_model().render_aabb) + #TODO jake-austin: Make this check whether the model inherits from a camera based model or a ray based model + if True: + pass + else: + camera_ray_bundle = camera.generate_rays(camera_indices=0, aabb_box=self.viewer.get_model().render_aabb) with TimeWriter(None, None, write=False) as vis_t: self.viewer.get_model().eval() @@ -142,12 +147,21 @@ class RenderStateMachine(threading.Thread): device=self.viewer.get_model().device, ) with background_color_override_context(background_color), torch.no_grad(): - outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) + if True: + outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera) + else: + outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) else: with torch.no_grad(): - outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) + if True: + outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera) + else: + outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) self.viewer.get_model().train() - num_rays = len(camera_ray_bundle) + if True: + num_rays = (camera.height * camera.width).item() + else: + num_rays = len(camera_ray_bundle) render_time = vis_t.duration if writer.is_initialized(): writer.put_time( diff --git a/nerfstudio/viewer_beta/render_state_machine.py b/nerfstudio/viewer_beta/render_state_machine.py index 6be9b3403..9ec840cc3 100644 --- a/nerfstudio/viewer_beta/render_state_machine.py +++ b/nerfstudio/viewer_beta/render_state_machine.py @@ -23,7 +23,8 @@ from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Tuple, get_args import torch -from nerfstudio.model_components.renderers import background_color_override_context +from nerfstudio.model_components.renderers import \ + background_color_override_context from nerfstudio.utils import colormaps, writer from nerfstudio.utils.writer import GLOBAL_BUFFER, EventName, TimeWriter from nerfstudio.viewer.server import viewer_utils @@ -133,7 +134,11 @@ class RenderStateMachine(threading.Thread): assert camera is not None, "render called before viewer connected" with self.viewer.train_lock if self.viewer.train_lock is not None else contextlib.nullcontext(): - camera_ray_bundle = camera.generate_rays(camera_indices=0, aabb_box=self.viewer.get_model().render_aabb) + #TODO jake-austin: Make this check whether the model inherits from a camera based model or a ray based model + if True: + pass + else: + camera_ray_bundle = camera.generate_rays(camera_indices=0, aabb_box=self.viewer.get_model().render_aabb) with TimeWriter(None, None, write=False) as vis_t: self.viewer.get_model().eval() @@ -148,12 +153,21 @@ class RenderStateMachine(threading.Thread): device=self.viewer.get_model().device, ) with background_color_override_context(background_color), torch.no_grad(): - outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) + if True: + outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera) + else: + outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) else: with torch.no_grad(): - outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) + if True: + outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera) + else: + outputs = self.viewer.get_model().get_outputs_for_camera_ray_bundle(camera_ray_bundle) self.viewer.get_model().train() - num_rays = len(camera_ray_bundle) + if True: + num_rays = (camera.height * camera.width).item() + else: + num_rays = len(camera_ray_bundle) render_time = vis_t.duration if writer.is_initialized(): writer.put_time( diff --git a/tests/test_train.py b/tests/test_train.py index a88283a59..45d8e348f 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -26,6 +26,7 @@ BLACKLIST = [ "neus", "generfacto", "neus-facto", + "gaussian-splatting", ]