diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index 869ad1176..d59db18c7 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -59,7 +59,7 @@ from nerfstudio.field_components.temporal_distortions import \ TemporalDistortionKind from nerfstudio.fields.sdf_field import SDFFieldConfig from nerfstudio.models.depth_nerfacto import DepthNerfactoModelConfig -from nerfstudio.models.gaussian_splatting import GaussianSplattingModelConfig +from nerfstudio.models.gaussian_splatting_inhouse import GaussianSplattingModelConfig from nerfstudio.models.generfacto import GenerfactoModelConfig from nerfstudio.models.instant_ngp import InstantNGPModelConfig from nerfstudio.models.mipnerf import MipNerfModel @@ -585,6 +585,54 @@ method_configs["neus-facto"] = TrainerConfig( vis="viewer", ) +# method_configs["gaussian-splatting"] = TrainerConfig( +# method_name="gaussian-splatting", +# steps_per_eval_image=10, +# steps_per_eval_batch=10, +# steps_per_save=2000, +# steps_per_eval_all_images=1000000, # set to a very large model so we don't eval with all images +# max_num_iterations=20001, +# mixed_precision=False, +# pipeline=VanillaPipelineConfig( +# datamanager=FullImageDatamanagerConfig( +# dataparser=BlenderDataParserConfig(), +# ), +# model=GaussianSplattingModelConfig(), +# ), +# optimizers={ +# "xyz": { +# "optimizer": AdamOptimizerConfig(lr=0.00016, eps=1e-15), +# "scheduler": ExponentialDecaySchedulerConfig( +# lr_pre_warmup=0.01 * 0.00016, +# lr_final=0.0000016, +# warmup_steps = 0, +# max_steps=30000, +# ), +# }, +# "f_dc": { +# "optimizer": AdamOptimizerConfig(lr=0.0025, eps=1e-15), +# "scheduler": None, +# }, +# "f_rest": { +# "optimizer": AdamOptimizerConfig(lr=0.0025, eps=1e-15), +# "scheduler": None, +# }, +# "opacity": { +# "optimizer": AdamOptimizerConfig(lr=0.05, eps=1e-15), +# "scheduler": None, +# }, +# "scaling": { +# "optimizer": AdamOptimizerConfig(lr=0.001, eps=1e-15), +# "scheduler": None, +# }, +# "rotation": { +# "optimizer": AdamOptimizerConfig(lr=0.001, eps=1e-15), +# "scheduler": None, +# }, +# }, +# viewer=ViewerConfig(num_rays_per_chunk=1 << 15), +# vis="viewer", +# ) method_configs["gaussian-splatting"] = TrainerConfig( method_name="gaussian-splatting", steps_per_eval_image=10, @@ -595,28 +643,18 @@ method_configs["gaussian-splatting"] = TrainerConfig( mixed_precision=False, pipeline=VanillaPipelineConfig( datamanager=FullImageDatamanagerConfig( - dataparser=BlenderDataParserConfig(), + dataparser=NerfstudioDataParserConfig(), ), model=GaussianSplattingModelConfig(), ), optimizers={ "xyz": { - "optimizer": AdamOptimizerConfig(lr=0.00016*5, eps=1e-15), + "optimizer": AdamOptimizerConfig(lr=0.00016, eps=1e-15), "scheduler": ExponentialDecaySchedulerConfig( - lr_pre_warmup=0.01 * 0.00016*5, - lr_final=0.0000016*5, - warmup_steps = 0, + lr_final=0.0000016, max_steps=30000, ), }, - "f_dc": { - "optimizer": AdamOptimizerConfig(lr=0.0025, eps=1e-15), - "scheduler": None, - }, - "f_rest": { - "optimizer": AdamOptimizerConfig(lr=0.0025, eps=1e-15), - "scheduler": None, - }, "opacity": { "optimizer": AdamOptimizerConfig(lr=0.05, eps=1e-15), "scheduler": None, diff --git a/nerfstudio/data/datamanagers/full_images_datamanager.py b/nerfstudio/data/datamanagers/full_images_datamanager.py index 82e17d031..cdb437bd9 100644 --- a/nerfstudio/data/datamanagers/full_images_datamanager.py +++ b/nerfstudio/data/datamanagers/full_images_datamanager.py @@ -39,31 +39,17 @@ from torch import Tensor, nn from torch.nn import Parameter from torch.utils.data.distributed import DistributedSampler from tqdm import tqdm -from typing_extensions import Literal, TypeVar +from typing_extensions import Literal -from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig from nerfstudio.cameras.cameras import Cameras, CameraType -from nerfstudio.cameras.rays import RayBundle -from nerfstudio.configs.base_config import InstantiateConfig -from nerfstudio.configs.config_utils import to_immutable_dict from nerfstudio.configs.dataparser_configs import AnnotatedDataParserUnion from nerfstudio.data.datamanagers.base_datamanager import (DataManager, - DataManagerConfig) + DataManagerConfig,TDataset) from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs from nerfstudio.data.dataparsers.blender_dataparser import \ BlenderDataParserConfig from nerfstudio.data.datasets.base_dataset import InputDataset -from nerfstudio.data.pixel_samplers import (PatchPixelSamplerConfig, - PixelSampler, PixelSamplerConfig) -from nerfstudio.data.scene_box import SceneBox -from nerfstudio.data.utils.dataloaders import (CacheDataloader, - FixedIndicesEvalDataloader, - RandIndicesEvalDataloader) -from nerfstudio.data.utils.nerfstudio_collate import nerfstudio_collate -from nerfstudio.engine.callbacks import (TrainingCallback, - TrainingCallbackAttributes) -from nerfstudio.model_components.ray_generators import RayGenerator -from nerfstudio.utils.misc import IterableWrapper, get_orig_class +from nerfstudio.utils.misc import get_orig_class from nerfstudio.utils.rich_utils import CONSOLE CONSOLE = Console(width=120) @@ -86,8 +72,6 @@ class FullImageDatamanagerConfig(DataManagerConfig): cache_images: Literal["no-cache", "cpu", "gpu"] = "cpu" """Whether to cache images in memory. If "numpy", caches as numpy arrays, if "torch", caches as torch tensors.""" -TDataset = TypeVar("TDataset", bound=InputDataset, default=InputDataset) - class FullImageDatamanager(DataManager, Generic[TDataset]): """ A datamanager that outputs full images and cameras instead of raybundles. This makes the diff --git a/nerfstudio/models/gaussian_splatting.py b/nerfstudio/models/gaussian_splatting.py index 677abcc84..ba911fb54 100644 --- a/nerfstudio/models/gaussian_splatting.py +++ b/nerfstudio/models/gaussian_splatting.py @@ -21,7 +21,6 @@ from __future__ import annotations from dataclasses import dataclass, field from typing import Any, Dict, List, Literal, Optional, Tuple, Type, Union -import diff_rast import matplotlib.pyplot as plt import numpy as np import torch diff --git a/nerfstudio/models/gaussian_splatting_inhouse.py b/nerfstudio/models/gaussian_splatting_inhouse.py new file mode 100644 index 000000000..259b7bb34 --- /dev/null +++ b/nerfstudio/models/gaussian_splatting_inhouse.py @@ -0,0 +1,217 @@ +# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +NeRF implementation that combines many recent advancements. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Literal, Optional, Tuple, Type, Union +import torch +from torch.nn import Parameter +from torchmetrics.functional import structural_similarity_index_measure +from torchmetrics.image import PeakSignalNoiseRatio +from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity + +from nerfstudio.cameras.cameras import Cameras +from nerfstudio.engine.callbacks import (TrainingCallback, + TrainingCallbackAttributes, + TrainingCallbackLocation) +from nerfstudio.models.base_model import Model, ModelConfig +import math +def random_quat_tensor(N, **kwargs): + u = torch.rand(N, **kwargs) + v = torch.rand(N, **kwargs) + w = torch.rand(N, **kwargs) + return torch.stack( + [ + torch.sqrt(1 - u) * torch.sin(2 * math.pi * v), + torch.sqrt(1 - u) * torch.cos(2 * math.pi * v), + torch.sqrt(u) * torch.sin(2 * math.pi * w), + torch.sqrt(u) * torch.sin(2 * math.pi * w), + ], + dim=-1, + ) + + +def identity_quat(N, **kwargs): + quat = torch.zeros(N, 4, **kwargs) + quat[:, 0] = 1 + return quat + + +def projection_matrix(znear, zfar, fovx, fovy, **kwargs): + t = math.tan(0.5 * fovy) + b = -t + r = math.tan(0.5 * fovx) + l = -r + n = znear + f = zfar + return torch.tensor( + [ + [2 * n / (r - l), 0.0, (r + l) / (r - l), 0.0], + [0.0, 2 * n / (t - b), (t + b) / (t - b), 0.0], + [0.0, 0.0, -(f + n) / (f - n), -2.0 * f * n / (f - n)], + [0.0, 0.0, -1.0, 0.0], + ], + **kwargs + ) + + +@dataclass +class GaussianSplattingModelConfig(ModelConfig): + """Gaussian Splatting Model Config""" + _target: Type = field(default_factory=lambda: GaussianSplattingModel) + num_points: int = 100000 + + +class GaussianSplattingModel(Model): + """Gaussian Splatting model + + TODO (jake-austin): Figure out how to print out on the training log in terminal the number of splats + + Args: + config: Gaussian Splatting configuration to instantiate model + """ + + config: GaussianSplattingModelConfig + + def populate_modules(self): + # TODO (jake-austin): clean this up, this is transplanted code across all the implementation functions + self.means = 2 * (torch.nn.Parameter(torch.rand(self.config.num_points, 3, device=self.device) - 0.5)) + self.scales = torch.nn.Parameter(torch.rand(self.config.num_points, 3, device=self.device)) + # self.quats = random_quat_tensor(self.num_points, device=self.device) + self.quats = torch.nn.Parameter(identity_quat(self.num_points, device=self.device)) + self.rgbs = torch.nn.Parameter(torch.rand(self.config.num_points, 3, device=self.device)) + self.opacities = torch.nn.Parameter(0.5 * torch.ones(self.config.num_points, 1, device=self.device)) + + # metrics + self.psnr = PeakSignalNoiseRatio(data_range=1.0) + self.ssim = structural_similarity_index_measure + self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True) + + def get_training_callbacks( + self, training_callback_attributes: TrainingCallbackAttributes + ) -> List[TrainingCallback]: + + return [] + + def get_param_groups(self) -> Dict[str, List[Parameter]]: + """Obtain the parameter groups for the optimizers + + Returns: + Mapping of different parameter groups + """ + return { + "xyz": [self.means], + "color": [self.rgbs], + "opacity": [self.opacities], + "scaling": [self.scales], + "rotation": [self.quats], + } + + def get_outputs(self, ray_bundle: Cameras) -> Dict[str, Union[torch.Tensor, List]]: + """Takes in a Ray Bundle and returns a dictionary of outputs. + + TODO (jake-austin): use the new homebrew nerfstudio gaussian rasterization code instead + + Args: + ray_bundle: Input bundle of rays. This raybundle should have all the + needed information to compute the outputs. + + Returns: + Outputs of model. (ie. rendered colors) + """ + + #print(f"Num Points: {self._xyz.shape[0]}") + + + camera = ray_bundle.reshape(()) + assert camera.shape == () + R = camera.camera_to_worlds[..., :3, :3].squeeze() + R[:,0] = -R[:,0] + T = (R.T @ camera.camera_to_worlds[..., :3, 3:4]).squeeze() + # return {"rgb": rgb} + + + def get_metrics_dict(self, outputs, batch) -> Dict[str, torch.Tensor]: + """Compute and returns metrics. + + Args: + outputs: the output to compute loss dict to + batch: ground truth batch corresponding to outputs + """ + + return {"Num Gaussians": self._xyz.shape[0]} + + def get_loss_dict(self, outputs, batch, metrics_dict=None) -> Dict[str, torch.Tensor]: + """Computes and returns the losses dict. + + Args: + outputs: the output to compute loss dict to + batch: ground truth batch corresponding to outputs + metrics_dict: dictionary of metrics, some of which we can use for loss + """ + # plt.imsave("test.png", np.concatenate([outputs["rgb"].detach().cpu().numpy(), batch["image"].detach().cpu().numpy()], axis=1)) + Ll1 = torch.nn.functional.l1_loss(batch['image'], outputs['rgb']) + return {"main_loss": Ll1} + + @torch.no_grad() + def get_outputs_for_camera_ray_bundle(self, camera_ray_bundle: Cameras) -> Dict[str, torch.Tensor]: + """Takes in camera parameters and computes the output of the model. + + Args: + camera_ray_bundle: ray bundle to calculate outputs over + """ + outs = self.get_outputs(camera_ray_bundle.to(self.device)) + outs["rgb"] = torch.clamp(outs["rgb"], 0.0, 1.0) + return outs + + def get_image_metrics_and_images( + self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor] + ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: + """Writes the test image outputs. + + Args: + image_idx: Index of the image. + step: Current step. + batch: Batch of data. + outputs: Outputs of the model. + + Returns: + A dictionary of metrics. + """ + gt_rgb = batch["image"].to(self.device) + predicted_rgb = outputs["rgb"] # Blended with background (black if random background) + + combined_rgb = torch.cat([gt_rgb, predicted_rgb], dim=1) + + # Switch images from [H, W, C] to [1, C, H, W] for metrics computations + gt_rgb = torch.moveaxis(gt_rgb, -1, 0)[None, ...] + predicted_rgb = torch.moveaxis(predicted_rgb, -1, 0)[None, ...] + + psnr = self.psnr(gt_rgb, predicted_rgb) + ssim = self.ssim(gt_rgb, predicted_rgb) + lpips = self.lpips(gt_rgb, predicted_rgb) + + # all of these metrics will be logged as scalars + metrics_dict = {"psnr": float(psnr.item()), "ssim": float(ssim)} # type: ignore + metrics_dict["lpips"] = float(lpips) + + images_dict = {"img": combined_rgb} + + return metrics_dict, images_dict + diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py index 388d3f9f9..6cd03fee9 100644 --- a/nerfstudio/pipelines/base_pipeline.py +++ b/nerfstudio/pipelines/base_pipeline.py @@ -259,6 +259,8 @@ class VanillaPipeline(Pipeline): self.datamanager: DataManager = config.datamanager.setup( device=device, test_mode=test_mode, world_size=world_size, local_rank=local_rank ) + print("Inside base pipeline") + import pdb;pdb.set_trace() self.datamanager.to(device) # TODO(ethan): get rid of scene_bounds from the model assert self.datamanager.train_dataset is not None, "Missing input dataset" diff --git a/nerfstudio/utils/gaussian_utils.py b/nerfstudio/utils/gaussian_utils.py index dcbed4367..b7f2a966f 100644 --- a/nerfstudio/utils/gaussian_utils.py +++ b/nerfstudio/utils/gaussian_utils.py @@ -1,365 +1,365 @@ -# -# Copyright (C) 2023, Inria -# GRAPHDECO research group, https://team.inria.fr/graphdeco -# All rights reserved. -# -# This software is free for non-commercial, research and evaluation use -# under the terms of the LICENSE.md file. -# -# For inquiries contact george.drettakis@inria.fr -# +# # +# # Copyright (C) 2023, Inria +# # GRAPHDECO research group, https://team.inria.fr/graphdeco +# # All rights reserved. +# # +# # This software is free for non-commercial, research and evaluation use +# # under the terms of the LICENSE.md file. +# # +# # For inquiries contact george.drettakis@inria.fr +# # -import math -import random -import sys -from datetime import datetime -from typing import NamedTuple +# import math +# import random +# import sys +# from datetime import datetime +# from typing import NamedTuple -import numpy as np -import torch -from diff_gaussian_rasterization import (GaussianRasterizationSettings, - GaussianRasterizer) -from torch import nn +# import numpy as np +# import torch +# from diff_gaussian_rasterization import (GaussianRasterizationSettings, +# GaussianRasterizer) +# from torch import nn -# TODO (jake-austin): remove this all outright as we move to our own homebrew gaussian splatting based rasterizer +# # TODO (jake-austin): remove this all outright as we move to our own homebrew gaussian splatting based rasterizer -def inverse_sigmoid(x): - return torch.log(x/(1-x)) +# def inverse_sigmoid(x): +# return torch.log(x/(1-x)) -def PILtoTorch(pil_image, resolution): - resized_image_PIL = pil_image.resize(resolution) - resized_image = torch.from_numpy(np.array(resized_image_PIL)) / 255.0 - if len(resized_image.shape) == 3: - return resized_image.permute(2, 0, 1) - else: - return resized_image.unsqueeze(dim=-1).permute(2, 0, 1) +# def PILtoTorch(pil_image, resolution): +# resized_image_PIL = pil_image.resize(resolution) +# resized_image = torch.from_numpy(np.array(resized_image_PIL)) / 255.0 +# if len(resized_image.shape) == 3: +# return resized_image.permute(2, 0, 1) +# else: +# return resized_image.unsqueeze(dim=-1).permute(2, 0, 1) -def strip_lowerdiag(L): - uncertainty = torch.zeros((L.shape[0], 6), dtype=torch.float, device="cuda") +# def strip_lowerdiag(L): +# uncertainty = torch.zeros((L.shape[0], 6), dtype=torch.float, device="cuda") - uncertainty[:, 0] = L[:, 0, 0] - uncertainty[:, 1] = L[:, 0, 1] - uncertainty[:, 2] = L[:, 0, 2] - uncertainty[:, 3] = L[:, 1, 1] - uncertainty[:, 4] = L[:, 1, 2] - uncertainty[:, 5] = L[:, 2, 2] - return uncertainty +# uncertainty[:, 0] = L[:, 0, 0] +# uncertainty[:, 1] = L[:, 0, 1] +# uncertainty[:, 2] = L[:, 0, 2] +# uncertainty[:, 3] = L[:, 1, 1] +# uncertainty[:, 4] = L[:, 1, 2] +# uncertainty[:, 5] = L[:, 2, 2] +# return uncertainty -def strip_symmetric(sym): - return strip_lowerdiag(sym) +# def strip_symmetric(sym): +# return strip_lowerdiag(sym) -def build_rotation(r): - norm = torch.sqrt(r[:,0]*r[:,0] + r[:,1]*r[:,1] + r[:,2]*r[:,2] + r[:,3]*r[:,3]) +# def build_rotation(r): +# norm = torch.sqrt(r[:,0]*r[:,0] + r[:,1]*r[:,1] + r[:,2]*r[:,2] + r[:,3]*r[:,3]) - q = r / norm[:, None] +# q = r / norm[:, None] - R = torch.zeros((q.size(0), 3, 3), device='cuda') +# R = torch.zeros((q.size(0), 3, 3), device='cuda') - r = q[:, 0] - x = q[:, 1] - y = q[:, 2] - z = q[:, 3] +# r = q[:, 0] +# x = q[:, 1] +# y = q[:, 2] +# z = q[:, 3] - R[:, 0, 0] = 1 - 2 * (y*y + z*z) - R[:, 0, 1] = 2 * (x*y - r*z) - R[:, 0, 2] = 2 * (x*z + r*y) - R[:, 1, 0] = 2 * (x*y + r*z) - R[:, 1, 1] = 1 - 2 * (x*x + z*z) - R[:, 1, 2] = 2 * (y*z - r*x) - R[:, 2, 0] = 2 * (x*z - r*y) - R[:, 2, 1] = 2 * (y*z + r*x) - R[:, 2, 2] = 1 - 2 * (x*x + y*y) - return R +# R[:, 0, 0] = 1 - 2 * (y*y + z*z) +# R[:, 0, 1] = 2 * (x*y - r*z) +# R[:, 0, 2] = 2 * (x*z + r*y) +# R[:, 1, 0] = 2 * (x*y + r*z) +# R[:, 1, 1] = 1 - 2 * (x*x + z*z) +# R[:, 1, 2] = 2 * (y*z - r*x) +# R[:, 2, 0] = 2 * (x*z - r*y) +# R[:, 2, 1] = 2 * (y*z + r*x) +# R[:, 2, 2] = 1 - 2 * (x*x + y*y) +# return R -def build_scaling_rotation(s, r): - L = torch.zeros((s.shape[0], 3, 3), dtype=torch.float, device="cuda") - R = build_rotation(r) +# def build_scaling_rotation(s, r): +# L = torch.zeros((s.shape[0], 3, 3), dtype=torch.float, device="cuda") +# R = build_rotation(r) - L[:,0,0] = s[:,0] - L[:,1,1] = s[:,1] - L[:,2,2] = s[:,2] +# L[:,0,0] = s[:,0] +# L[:,1,1] = s[:,1] +# L[:,2,2] = s[:,2] - L = R @ L - return L +# L = R @ L +# return L -def safe_state(silent): - old_f = sys.stdout - class F: - def __init__(self, silent): - self.silent = silent +# def safe_state(silent): +# old_f = sys.stdout +# class F: +# def __init__(self, silent): +# self.silent = silent - def write(self, x): - if not self.silent: - if x.endswith("\n"): - old_f.write(x.replace("\n", " [{}]\n".format(str(datetime.now().strftime("%d/%m %H:%M:%S"))))) - else: - old_f.write(x) +# def write(self, x): +# if not self.silent: +# if x.endswith("\n"): +# old_f.write(x.replace("\n", " [{}]\n".format(str(datetime.now().strftime("%d/%m %H:%M:%S"))))) +# else: +# old_f.write(x) - def flush(self): - old_f.flush() +# def flush(self): +# old_f.flush() - sys.stdout = F(silent) +# sys.stdout = F(silent) - random.seed(0) - np.random.seed(0) - torch.manual_seed(0) - torch.cuda.set_device(torch.device("cuda:0")) +# random.seed(0) +# np.random.seed(0) +# torch.manual_seed(0) +# torch.cuda.set_device(torch.device("cuda:0")) -def render_from_dict(data : dict, bg_color : torch.Tensor, scaling_modifier = 1.0, override_color = None): - """ - Render the scene. +# def render_from_dict(data : dict, bg_color : torch.Tensor, scaling_modifier = 1.0, override_color = None): +# """ +# Render the scene. - Background tensor (bg_color) must be on GPU! - """ - for k in ['active_sh_degree', 'max_sh_degree', 'xyz', 'scaling', 'rotation', 'features', 'opacity', 'FoVx', 'FoVy', 'image_height', 'image_width', 'world_view_transform', 'full_proj_transform', 'camera_center']: - assert k in data, f'Required key {k} not in {data.keys()}' +# Background tensor (bg_color) must be on GPU! +# """ +# for k in ['active_sh_degree', 'max_sh_degree', 'xyz', 'scaling', 'rotation', 'features', 'opacity', 'FoVx', 'FoVy', 'image_height', 'image_width', 'world_view_transform', 'full_proj_transform', 'camera_center']: +# assert k in data, f'Required key {k} not in {data.keys()}' - # Create zero tensor. We will use it to make pytorch return gradients of the 2D (screen-space) means - screenspace_points = torch.zeros_like(data['xyz'], dtype=data['xyz'].dtype, requires_grad=True, device="cuda") + 0 - try: - screenspace_points.retain_grad() - except: - pass +# # Create zero tensor. We will use it to make pytorch return gradients of the 2D (screen-space) means +# screenspace_points = torch.zeros_like(data['xyz'], dtype=data['xyz'].dtype, requires_grad=True, device="cuda") + 0 +# try: +# screenspace_points.retain_grad() +# except: +# pass - # Set up rasterization configuration - tanfovx = torch.tan(data['FoVx'] * 0.5).item() - tanfovy = torch.tan(data['FoVy'] * 0.5).item() +# # Set up rasterization configuration +# tanfovx = torch.tan(data['FoVx'] * 0.5).item() +# tanfovy = torch.tan(data['FoVy'] * 0.5).item() - raster_settings = GaussianRasterizationSettings( - image_height=int(data['image_height']), - image_width=int(data['image_width']), - tanfovx=tanfovx, - tanfovy=tanfovy, - bg=bg_color, - scale_modifier=scaling_modifier, - viewmatrix=data['world_view_transform'], - projmatrix=data['full_proj_transform'], - sh_degree=data['active_sh_degree'], - campos=data['camera_center'], - prefiltered=False, - debug=False - ) +# raster_settings = GaussianRasterizationSettings( +# image_height=int(data['image_height']), +# image_width=int(data['image_width']), +# tanfovx=tanfovx, +# tanfovy=tanfovy, +# bg=bg_color, +# scale_modifier=scaling_modifier, +# viewmatrix=data['world_view_transform'], +# projmatrix=data['full_proj_transform'], +# sh_degree=data['active_sh_degree'], +# campos=data['camera_center'], +# prefiltered=False, +# debug=False +# ) - rasterizer = GaussianRasterizer(raster_settings=raster_settings) +# rasterizer = GaussianRasterizer(raster_settings=raster_settings) - means3D = data['xyz'] - means2D = screenspace_points - opacity = data['opacity'] +# means3D = data['xyz'] +# means2D = screenspace_points +# opacity = data['opacity'] - # If precomputed 3d covariance is provided, use it. If not, then it will be computed from - # scaling / rotation by the rasterizer. - scales = None - rotations = None - cov3D_precomp = None - # if pipe.compute_cov3D_python: - # cov3D_precomp = pc.get_covariance(scaling_modifier) - # else: - scales = data['scaling'] - rotations = data['rotation'] +# # If precomputed 3d covariance is provided, use it. If not, then it will be computed from +# # scaling / rotation by the rasterizer. +# scales = None +# rotations = None +# cov3D_precomp = None +# # if pipe.compute_cov3D_python: +# # cov3D_precomp = pc.get_covariance(scaling_modifier) +# # else: +# scales = data['scaling'] +# rotations = data['rotation'] - # If precomputed colors are provided, use them. Otherwise, if it is desired to precompute colors - # from SHs in Python, do it. If not, then SH -> RGB conversion will be done by rasterizer. - shs = None - colors_precomp = None - if colors_precomp is None: - shs = data['features'] - else: +# # If precomputed colors are provided, use them. Otherwise, if it is desired to precompute colors +# # from SHs in Python, do it. If not, then SH -> RGB conversion will be done by rasterizer. +# shs = None +# colors_precomp = None +# if colors_precomp is None: +# shs = data['features'] +# else: - colors_precomp = override_color +# colors_precomp = override_color - # Rasterize visible Gaussians to image, obtain their radii (on screen). - rendered_image, radii = rasterizer( - means3D = means3D, - means2D = means2D, - shs = shs, - colors_precomp = colors_precomp, - opacities = opacity, - scales = scales, - rotations = rotations, - cov3D_precomp = cov3D_precomp) +# # Rasterize visible Gaussians to image, obtain their radii (on screen). +# rendered_image, radii = rasterizer( +# means3D = means3D, +# means2D = means2D, +# shs = shs, +# colors_precomp = colors_precomp, +# opacities = opacity, +# scales = scales, +# rotations = rotations, +# cov3D_precomp = cov3D_precomp) - # Those Gaussians that were frustum culled or had a radius of 0 were not visible. - # They will be excluded from value updates used in the splitting criteria. - return {"render": rendered_image, - "viewspace_points": screenspace_points, - "visibility_filter" : radii > 0, - "radii": radii} +# # Those Gaussians that were frustum culled or had a radius of 0 were not visible. +# # They will be excluded from value updates used in the splitting criteria. +# return {"render": rendered_image, +# "viewspace_points": screenspace_points, +# "visibility_filter" : radii > 0, +# "radii": radii} -def render_accumulation_from_dict(viewpoint_camera, pc : dict, bg_color : torch.Tensor, scaling_modifier = 1.0): - """ - TODO (jake-austin): double check that this is actually equivalent to getting the accumulation - """ - override_color = torch.ones_like(pc['xyz'], dtype=pc['xyz'].dtype, requires_grad=True, device="cuda") + 0 - return render_from_dict( - viewpoint_camera, - pc, - bg_color, - scaling_modifier, - override_color=override_color - ) +# def render_accumulation_from_dict(viewpoint_camera, pc : dict, bg_color : torch.Tensor, scaling_modifier = 1.0): +# """ +# TODO (jake-austin): double check that this is actually equivalent to getting the accumulation +# """ +# override_color = torch.ones_like(pc['xyz'], dtype=pc['xyz'].dtype, requires_grad=True, device="cuda") + 0 +# return render_from_dict( +# viewpoint_camera, +# pc, +# bg_color, +# scaling_modifier, +# override_color=override_color +# ) -def inverse_sigmoid(x): - return torch.log(x/(1-x)) +# def inverse_sigmoid(x): +# return torch.log(x/(1-x)) -C0 = 0.28209479177387814 +# C0 = 0.28209479177387814 -def RGB2SH(rgb): - return (rgb - 0.5) / C0 +# def RGB2SH(rgb): +# return (rgb - 0.5) / C0 -def SH2RGB(sh): - return sh * C0 + 0.5 +# def SH2RGB(sh): +# return sh * C0 + 0.5 -def getWorld2View(R, t): - Rt = torch.zeros((4, 4)) - Rt[:3, :3] = R.transpose() - Rt[:3, 3] = t - Rt[3, 3] = 1.0 - return torch.float32(Rt) +# def getWorld2View(R, t): +# Rt = torch.zeros((4, 4)) +# Rt[:3, :3] = R.transpose() +# Rt[:3, 3] = t +# Rt[3, 3] = 1.0 +# return torch.float32(Rt) -def getWorld2View2(R, t, translate=torch.tensor([.0, .0, .0]), scale=1.0): - Rt = torch.zeros((4, 4)).to(R.device) - Rt[:3, :3] = R.transpose(-1,-2) - Rt[:3, 3] = t - Rt[3, 3] = 1.0 +# def getWorld2View2(R, t, translate=torch.tensor([.0, .0, .0]), scale=1.0): +# Rt = torch.zeros((4, 4)).to(R.device) +# Rt[:3, :3] = R.transpose(-1,-2) +# Rt[:3, 3] = t +# Rt[3, 3] = 1.0 - C2W = torch.linalg.inv(Rt) - cam_center = C2W[:3, 3] +# C2W = torch.linalg.inv(Rt) +# cam_center = C2W[:3, 3] - cam_center = (cam_center + translate.to(R.device)) * scale - C2W[:3, 3] = cam_center - Rt = torch.linalg.inv(C2W) - return Rt +# cam_center = (cam_center + translate.to(R.device)) * scale +# C2W[:3, 3] = cam_center +# Rt = torch.linalg.inv(C2W) +# return Rt -def getProjectionMatrix(znear, zfar, fovX, fovY): - tanHalfFovY = torch.tan((fovY / 2)) - tanHalfFovX = torch.tan((fovX / 2)) +# def getProjectionMatrix(znear, zfar, fovX, fovY): +# tanHalfFovY = torch.tan((fovY / 2)) +# tanHalfFovX = torch.tan((fovX / 2)) - top = tanHalfFovY * znear - bottom = -top - right = tanHalfFovX * znear - left = -right +# top = tanHalfFovY * znear +# bottom = -top +# right = tanHalfFovX * znear +# left = -right - P = torch.zeros(4, 4).to(fovX.device) +# P = torch.zeros(4, 4).to(fovX.device) - z_sign = 1.0 +# z_sign = 1.0 - P[0, 0] = 2.0 * znear / (right - left) - P[1, 1] = 2.0 * znear / (top - bottom) - P[0, 2] = (right + left) / (right - left) - P[1, 2] = (top + bottom) / (top - bottom) - P[3, 2] = z_sign - P[2, 2] = z_sign * zfar / (zfar - znear) - P[2, 3] = -(zfar * znear) / (zfar - znear) - return P +# P[0, 0] = 2.0 * znear / (right - left) +# P[1, 1] = 2.0 * znear / (top - bottom) +# P[0, 2] = (right + left) / (right - left) +# P[1, 2] = (top + bottom) / (top - bottom) +# P[3, 2] = z_sign +# P[2, 2] = z_sign * zfar / (zfar - znear) +# P[2, 3] = -(zfar * znear) / (zfar - znear) +# return P -def fov2focal(fov, pixels): - return pixels / (2 * torch.tan(fov / 2)) +# def fov2focal(fov, pixels): +# return pixels / (2 * torch.tan(fov / 2)) -def focal2fov(focal, pixels): - return 2*torch.atan(pixels/(2*focal)) +# def focal2fov(focal, pixels): +# return 2*torch.atan(pixels/(2*focal)) -class Camera(nn.Module): - def __init__(self, R, T, FoVx, FoVy, height, width, data_device = "cuda"): - super(Camera, self).__init__() +# class Camera(nn.Module): +# def __init__(self, R, T, FoVx, FoVy, height, width, data_device = "cuda"): +# super(Camera, self).__init__() - self.R = R - self.T = T - self.FoVx = FoVx - self.FoVy = FoVy +# self.R = R +# self.T = T +# self.FoVx = FoVx +# self.FoVy = FoVy - try: - self.data_device = torch.device(data_device) - except Exception as e: - print(e) - print(f"[Warning] Custom device {data_device} failed, fallback to default cuda device" ) - self.data_device = torch.device("cuda") +# try: +# self.data_device = torch.device(data_device) +# except Exception as e: +# print(e) +# print(f"[Warning] Custom device {data_device} failed, fallback to default cuda device" ) +# self.data_device = torch.device("cuda") - self.image_width = width - self.image_height = height +# self.image_width = width +# self.image_height = height - self.zfar = 100.0 - self.znear = 0.01 +# self.zfar = 100.0 +# self.znear = 0.01 - self.trans =torch.tensor([0.0, 0.0, 0.0]).to(self.data_device) - self.scale = 1.0 +# self.trans =torch.tensor([0.0, 0.0, 0.0]).to(self.data_device) +# self.scale = 1.0 - self.world_view_transform = getWorld2View2(R, T, self.trans, self.scale).transpose(0, 1).to(self.data_device) - self.projection_matrix = getProjectionMatrix(znear=self.znear, zfar=self.zfar, fovX=self.FoVx, fovY=self.FoVy).transpose(0,1).to(self.data_device) - self.full_proj_transform = (self.world_view_transform.unsqueeze(0).bmm(self.projection_matrix.unsqueeze(0))).squeeze(0) - self.camera_center = self.world_view_transform.inverse()[3, :3] +# self.world_view_transform = getWorld2View2(R, T, self.trans, self.scale).transpose(0, 1).to(self.data_device) +# self.projection_matrix = getProjectionMatrix(znear=self.znear, zfar=self.zfar, fovX=self.FoVx, fovY=self.FoVy).transpose(0,1).to(self.data_device) +# self.full_proj_transform = (self.world_view_transform.unsqueeze(0).bmm(self.projection_matrix.unsqueeze(0))).squeeze(0) +# self.camera_center = self.world_view_transform.inverse()[3, :3] -class MiniCam: - def __init__(self, width, height, fovy, fovx, znear, zfar, world_view_transform, full_proj_transform): - self.image_width = width - self.image_height = height - self.FoVy = fovy - self.FoVx = fovx - self.znear = znear - self.zfar = zfar - self.world_view_transform = world_view_transform - self.full_proj_transform = full_proj_transform - view_inv = torch.inverse(self.world_view_transform) - self.camera_center = view_inv[3][:3] +# class MiniCam: +# def __init__(self, width, height, fovy, fovx, znear, zfar, world_view_transform, full_proj_transform): +# self.image_width = width +# self.image_height = height +# self.FoVy = fovy +# self.FoVx = fovx +# self.znear = znear +# self.zfar = zfar +# self.world_view_transform = world_view_transform +# self.full_proj_transform = full_proj_transform +# view_inv = torch.inverse(self.world_view_transform) +# self.camera_center = view_inv[3][:3] -from math import exp +# from math import exp -import torch -import torch.nn.functional as F -from torch.autograd import Variable +# import torch +# import torch.nn.functional as F +# from torch.autograd import Variable -def l1_loss(network_output, gt): - return torch.abs((network_output - gt)).mean() +# def l1_loss(network_output, gt): +# return torch.abs((network_output - gt)).mean() -def l2_loss(network_output, gt): - return ((network_output - gt) ** 2).mean() +# def l2_loss(network_output, gt): +# return ((network_output - gt) ** 2).mean() -def gaussian(window_size, sigma): - gauss = torch.Tensor([exp(-(x - window_size // 2) ** 2 / float(2 * sigma ** 2)) for x in range(window_size)]) - return gauss / gauss.sum() +# def gaussian(window_size, sigma): +# gauss = torch.Tensor([exp(-(x - window_size // 2) ** 2 / float(2 * sigma ** 2)) for x in range(window_size)]) +# return gauss / gauss.sum() -def create_window(window_size, channel): - _1D_window = gaussian(window_size, 1.5).unsqueeze(1) - _2D_window = _1D_window.mm(_1D_window.t()).float().unsqueeze(0).unsqueeze(0) - window = Variable(_2D_window.expand(channel, 1, window_size, window_size).contiguous()) - return window +# def create_window(window_size, channel): +# _1D_window = gaussian(window_size, 1.5).unsqueeze(1) +# _2D_window = _1D_window.mm(_1D_window.t()).float().unsqueeze(0).unsqueeze(0) +# window = Variable(_2D_window.expand(channel, 1, window_size, window_size).contiguous()) +# return window -def ssim(img1, img2, window_size=11, size_average=True): - channel = img1.size(-3) - window = create_window(window_size, channel) +# def ssim(img1, img2, window_size=11, size_average=True): +# channel = img1.size(-3) +# window = create_window(window_size, channel) - if img1.is_cuda: - window = window.cuda(img1.get_device()) - window = window.type_as(img1) +# if img1.is_cuda: +# window = window.cuda(img1.get_device()) +# window = window.type_as(img1) - return _ssim(img1, img2, window, window_size, channel, size_average) +# return _ssim(img1, img2, window, window_size, channel, size_average) -def _ssim(img1, img2, window, window_size, channel, size_average=True): - mu1 = F.conv2d(img1, window, padding=window_size // 2, groups=channel) - mu2 = F.conv2d(img2, window, padding=window_size // 2, groups=channel) +# def _ssim(img1, img2, window, window_size, channel, size_average=True): +# mu1 = F.conv2d(img1, window, padding=window_size // 2, groups=channel) +# mu2 = F.conv2d(img2, window, padding=window_size // 2, groups=channel) - mu1_sq = mu1.pow(2) - mu2_sq = mu2.pow(2) - mu1_mu2 = mu1 * mu2 +# mu1_sq = mu1.pow(2) +# mu2_sq = mu2.pow(2) +# mu1_mu2 = mu1 * mu2 - sigma1_sq = F.conv2d(img1 * img1, window, padding=window_size // 2, groups=channel) - mu1_sq - sigma2_sq = F.conv2d(img2 * img2, window, padding=window_size // 2, groups=channel) - mu2_sq - sigma12 = F.conv2d(img1 * img2, window, padding=window_size // 2, groups=channel) - mu1_mu2 +# sigma1_sq = F.conv2d(img1 * img1, window, padding=window_size // 2, groups=channel) - mu1_sq +# sigma2_sq = F.conv2d(img2 * img2, window, padding=window_size // 2, groups=channel) - mu2_sq +# sigma12 = F.conv2d(img1 * img2, window, padding=window_size // 2, groups=channel) - mu1_mu2 - C1 = 0.01 ** 2 - C2 = 0.03 ** 2 +# C1 = 0.01 ** 2 +# C2 = 0.03 ** 2 - ssim_map = ((2 * mu1_mu2 + C1) * (2 * sigma12 + C2)) / ((mu1_sq + mu2_sq + C1) * (sigma1_sq + sigma2_sq + C2)) +# ssim_map = ((2 * mu1_mu2 + C1) * (2 * sigma12 + C2)) / ((mu1_sq + mu2_sq + C1) * (sigma1_sq + sigma2_sq + C2)) - if size_average: - return ssim_map.mean() - else: - return ssim_map.mean(1).mean(1).mean(1) +# if size_average: +# return ssim_map.mean() +# else: +# return ssim_map.mean(1).mean(1).mean(1)