mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-20 06:35:36 +08:00
tensorf (#586)
* tensorf wip * model changes * tuning model * changing imports * merging * changing rff encoding to nerfencoding, basis matrix should be properly updated, changed density activation, lr scheduler * debugging tensorf * debugging tensorf * tensorf debugging * tensorf debugging * tensorf debugging * tensorf debug * debugging tensorf, fixed bug with collider * tuning model, ray sampling * adding thresholding for ray computations * removing unused imports * making more params configurable in config, adding spherical harmonics * removing scene contraction, adding arg docstrings * fixing style * Fixing bug Co-authored-by: Matthew Tancik <tancik@berkeley.edu>
This commit is contained in:
1 parent
35610cb9e2
commit
7106f42fcc
4 files changed
+444
-3
No files matched your search
@@ -23,7 +23,12 @@ from typing import Dict
|
||||
import tyro
|
||||
|
||||
from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig
|
||||
from nerfstudio.configs.base_config import Config, TrainerConfig, ViewerConfig
|
||||
from nerfstudio.configs.base_config import (
|
||||
Config,
|
||||
SchedulerConfig,
|
||||
TrainerConfig,
|
||||
ViewerConfig,
|
||||
)
|
||||
from nerfstudio.data.datamanagers import VanillaDataManagerConfig
|
||||
from nerfstudio.data.dataparsers.blender_dataparser import BlenderDataParserConfig
|
||||
from nerfstudio.data.dataparsers.friends_dataparser import FriendsDataParserConfig
|
||||
@@ -34,6 +39,7 @@ from nerfstudio.models.instant_ngp import InstantNGPModelConfig
|
||||
from nerfstudio.models.mipnerf import MipNerfModel
|
||||
from nerfstudio.models.nerfacto import NerfactoModelConfig
|
||||
from nerfstudio.models.semantic_nerfw import SemanticNerfWModelConfig
|
||||
from nerfstudio.models.tensorf import TensoRFModelConfig
|
||||
from nerfstudio.models.vanilla_nerf import NeRFModel
|
||||
from nerfstudio.pipelines.base_pipeline import VanillaPipelineConfig
|
||||
from nerfstudio.pipelines.dynamic_batch import DynamicBatchPipelineConfig
|
||||
@@ -45,6 +51,7 @@ descriptions = {
|
||||
"mipnerf": "High quality model for bounded scenes. (slow)",
|
||||
"semantic-nerfw": "Predicts semantic segmentations and filters out transient objects.",
|
||||
"vanilla-nerf": "Original NeRF model. (slow)",
|
||||
"tensorf": "tensorf",
|
||||
}
|
||||
|
||||
method_configs["nerfacto"] = Config(
|
||||
@@ -155,6 +162,26 @@ method_configs["vanilla-nerf"] = Config(
|
||||
},
|
||||
)
|
||||
|
||||
method_configs["tensorf"] = Config(
|
||||
method_name="tensorf",
|
||||
trainer=TrainerConfig(mixed_precision=False),
|
||||
pipeline=VanillaPipelineConfig(
|
||||
datamanager=VanillaDataManagerConfig(
|
||||
dataparser=BlenderDataParserConfig(),
|
||||
),
|
||||
model=TensoRFModelConfig(),
|
||||
),
|
||||
optimizers={
|
||||
"fields": {
|
||||
"optimizer": AdamOptimizerConfig(lr=0.001),
|
||||
"scheduler": SchedulerConfig(lr_final=0.0001, max_steps=30000),
|
||||
},
|
||||
"encodings": {
|
||||
"optimizer": AdamOptimizerConfig(lr=0.02),
|
||||
"scheduler": SchedulerConfig(lr_final=0.002, max_steps=30000),
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
AnnotatedBaseConfigUnion = tyro.conf.SuppressFixed[ # Don't show unparseable (fixed) arguments in helptext.
|
||||
tyro.extras.subcommand_type_from_defaults(defaults=method_configs, descriptions=descriptions)
|
||||
|
||||
@@ -416,7 +416,6 @@ class TensorVMEncoding(Encoding):
|
||||
self.resolution = resolution
|
||||
self.num_components = num_components
|
||||
|
||||
# TODO(terrance): Learning rates should be different for these
|
||||
self.plane_coef = nn.Parameter(init_scale * torch.randn((3, num_components, resolution, resolution)))
|
||||
self.line_coef = nn.Parameter(init_scale * torch.randn((3, num_components, resolution, 1)))
|
||||
|
||||
@@ -424,7 +423,6 @@ class TensorVMEncoding(Encoding):
|
||||
return self.num_components * 3
|
||||
|
||||
def forward(self, in_tensor: TensorType["bs":..., "input_dim"]) -> TensorType["bs":..., "output_dim"]:
|
||||
in_tensor = in_tensor / 4.0 + 0.5
|
||||
plane_coord = torch.stack([in_tensor[..., [0, 1]], in_tensor[..., [0, 2]], in_tensor[..., [1, 2]]]) # [3,...,2]
|
||||
line_coord = torch.stack([in_tensor[..., 2], in_tensor[..., 1], in_tensor[..., 0]]) # [3, ...]
|
||||
line_coord = torch.stack([torch.zeros_like(line_coord), line_coord], dim=-1) # [3, ...., 2]
|
||||
@@ -452,6 +450,7 @@ class TensorVMEncoding(Encoding):
|
||||
self.plane_coef.data, size=(resolution, resolution), mode="bilinear", align_corners=True
|
||||
)
|
||||
line_coef = F.interpolate(self.line_coef.data, size=(resolution, 1), mode="bilinear", align_corners=True)
|
||||
|
||||
# TODO(ethan): are these torch.nn.Parameters needed?
|
||||
self.plane_coef, self.line_coef = torch.nn.Parameter(plane_coef), torch.nn.Parameter(line_coef)
|
||||
self.resolution = resolution
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
# Copyright 2022 The Plenoptix Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""TensoRF Field"""
|
||||
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.nn.parameter import Parameter
|
||||
from torchtyping import TensorType
|
||||
|
||||
from nerfstudio.cameras.rays import RaySamples
|
||||
from nerfstudio.data.scene_box import SceneBox
|
||||
from nerfstudio.field_components.encodings import Encoding, Identity, SHEncoding
|
||||
from nerfstudio.field_components.field_heads import FieldHeadNames, RGBFieldHead
|
||||
from nerfstudio.field_components.mlp import MLP
|
||||
from nerfstudio.fields.base_field import Field
|
||||
|
||||
|
||||
class TensoRFField(Field):
|
||||
"""TensoRF Field"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
aabb,
|
||||
# the aabb bounding box of the dataset
|
||||
feature_encoding: Encoding = Identity(in_dim=3),
|
||||
# the encoding method used for appearance encoding outputs
|
||||
direction_encoding: Encoding = Identity(in_dim=3),
|
||||
# the encoding method used for ray direction
|
||||
density_encoding: Encoding = Identity(in_dim=3),
|
||||
# the tensor encoding method used for scene density
|
||||
color_encoding: Encoding = Identity(in_dim=3),
|
||||
# the tensor encoding method used for scene color
|
||||
appearance_dim: int = 27,
|
||||
# the number of dimensions for the appearance embedding
|
||||
head_mlp_num_layers: int = 2,
|
||||
# number of layers for the MLP
|
||||
head_mlp_layer_width: int = 128,
|
||||
# layer width for the MLP
|
||||
use_sh: bool = False,
|
||||
# whether to use spherical harmonics as the feature decoding function
|
||||
sh_levels: int = 2,
|
||||
# number of levels to use for spherical harmonics
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.aabb = Parameter(aabb, requires_grad=False)
|
||||
self.feature_encoding = feature_encoding
|
||||
self.direction_encoding = direction_encoding
|
||||
self.density_encoding = density_encoding
|
||||
self.color_encoding = color_encoding
|
||||
|
||||
self.mlp_head = MLP(
|
||||
in_dim=appearance_dim + 3 + self.direction_encoding.get_out_dim() + self.feature_encoding.get_out_dim(),
|
||||
num_layers=head_mlp_num_layers,
|
||||
layer_width=head_mlp_layer_width,
|
||||
activation=nn.ReLU(),
|
||||
)
|
||||
|
||||
self.use_sh = use_sh
|
||||
|
||||
if self.use_sh:
|
||||
self.sh = SHEncoding(sh_levels)
|
||||
self.B = nn.Linear(
|
||||
in_features=self.color_encoding.get_out_dim(), out_features=3 * self.sh.get_out_dim(), bias=False
|
||||
)
|
||||
else:
|
||||
self.B = nn.Linear(in_features=self.color_encoding.get_out_dim(), out_features=appearance_dim, bias=False)
|
||||
|
||||
self.field_output_rgb = RGBFieldHead(in_dim=self.mlp_head.get_out_dim(), activation=nn.Sigmoid())
|
||||
|
||||
def get_density(self, ray_samples: RaySamples):
|
||||
positions = SceneBox.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb)
|
||||
density = self.density_encoding(positions)
|
||||
density_enc = torch.sum(density, dim=-1)[:, :, None]
|
||||
relu = torch.nn.ReLU()
|
||||
density_enc = relu(density_enc)
|
||||
return density_enc
|
||||
|
||||
def get_outputs(self, ray_samples: RaySamples, density_embedding: Optional[TensorType] = None) -> TensorType:
|
||||
d = ray_samples.frustums.directions
|
||||
positions = SceneBox.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb)
|
||||
rgb_features = self.color_encoding(positions)
|
||||
rgb_features = self.B(rgb_features)
|
||||
|
||||
d_encoded = self.direction_encoding(d)
|
||||
rgb_features_encoded = self.feature_encoding(rgb_features)
|
||||
|
||||
if self.use_sh:
|
||||
sh_mult = self.sh(d)[:, :, None]
|
||||
rgb_sh = rgb_features.view(sh_mult.shape[0], sh_mult.shape[1], 3, sh_mult.shape[-1])
|
||||
rgb = torch.relu(torch.sum(sh_mult * rgb_sh, dim=-1) + 0.5)
|
||||
else:
|
||||
out = self.mlp_head(torch.cat([rgb_features, d, rgb_features_encoded, d_encoded], dim=-1)) # type: ignore
|
||||
rgb = self.field_output_rgb(out)
|
||||
|
||||
return rgb
|
||||
|
||||
def forward(
|
||||
self, ray_samples: RaySamples, mask: Optional[TensorType] = None, bg_color: Optional[TensorType] = None
|
||||
):
|
||||
if mask is not None and bg_color is not None:
|
||||
base_density = torch.zeros(ray_samples.shape)[:, :, None].to(mask.device)
|
||||
base_rgb = bg_color.repeat(ray_samples[:, :, None].shape)
|
||||
if mask.any():
|
||||
input_rays = ray_samples[mask, :]
|
||||
density = self.get_density(input_rays)
|
||||
rgb = self.get_outputs(input_rays, None)
|
||||
|
||||
base_density[mask] = density
|
||||
base_rgb[mask] = rgb
|
||||
|
||||
base_density.requires_grad_()
|
||||
base_rgb.requires_grad_()
|
||||
|
||||
density = base_density
|
||||
rgb = base_rgb
|
||||
else:
|
||||
density = self.get_density(ray_samples)
|
||||
rgb = self.get_outputs(ray_samples, None)
|
||||
|
||||
return {FieldHeadNames.DENSITY: density, FieldHeadNames.RGB: rgb}
|
||||
@@ -0,0 +1,280 @@
|
||||
# Copyright 2022 The Nerfstudio Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
TensorRF implementation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Tuple, Type
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch.nn import Parameter
|
||||
from torchmetrics import PeakSignalNoiseRatio
|
||||
from torchmetrics.functional import structural_similarity_index_measure
|
||||
from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity
|
||||
|
||||
from nerfstudio.cameras.rays import RayBundle
|
||||
from nerfstudio.configs.config_utils import to_immutable_dict
|
||||
from nerfstudio.engine.callbacks import (
|
||||
TrainingCallback,
|
||||
TrainingCallbackAttributes,
|
||||
TrainingCallbackLocation,
|
||||
)
|
||||
from nerfstudio.field_components.encodings import NeRFEncoding, TensorVMEncoding
|
||||
from nerfstudio.field_components.field_heads import FieldHeadNames
|
||||
from nerfstudio.fields.tensorf_field import TensoRFField
|
||||
from nerfstudio.model_components.losses import MSELoss
|
||||
from nerfstudio.model_components.ray_samplers import PDFSampler, UniformSampler
|
||||
from nerfstudio.model_components.renderers import (
|
||||
AccumulationRenderer,
|
||||
DepthRenderer,
|
||||
RGBRenderer,
|
||||
)
|
||||
from nerfstudio.model_components.scene_colliders import AABBBoxCollider
|
||||
from nerfstudio.models.base_model import Model, VanillaModelConfig
|
||||
from nerfstudio.utils import colormaps, colors, misc
|
||||
|
||||
|
||||
@dataclass
|
||||
class TensoRFModelConfig(VanillaModelConfig):
|
||||
"""TensoRF model config"""
|
||||
|
||||
_target: Type = field(default_factory=lambda: TensoRFModel)
|
||||
"""target class to instantiate"""
|
||||
init_resolution: int = 128
|
||||
"""initial render resolution"""
|
||||
final_resolution: int = 300
|
||||
"""final render resolution"""
|
||||
upsampling_iters: Tuple[int, ...] = (2000, 3000, 4000, 5500, 7000)
|
||||
"""specifies a list of iteration step numbers to perform upsampling"""
|
||||
loss_coefficients: Dict[str, float] = to_immutable_dict({"rgb_loss": 1.0})
|
||||
"""Loss specific weights."""
|
||||
num_samples: int = 256
|
||||
"""Number of samples in field evaluation"""
|
||||
num_den_components: int = 16
|
||||
"""Number of components in density encoding"""
|
||||
num_color_components: int = 48
|
||||
"""Number of components in color encoding"""
|
||||
appearance_dim: int = 27
|
||||
"""Number of channels for color encoding"""
|
||||
|
||||
|
||||
class TensoRFModel(Model):
|
||||
"""TensoRF Model
|
||||
|
||||
Args:
|
||||
config: TensoRF configuration to instantiate model
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: TensoRFModelConfig,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.init_resolution = config.init_resolution
|
||||
self.upsampling_iters = config.upsampling_iters
|
||||
self.num_den_components = config.num_den_components
|
||||
self.num_color_components = config.num_color_components
|
||||
self.appearance_dim = config.appearance_dim
|
||||
self.upsampling_steps = (
|
||||
np.round(
|
||||
np.exp(
|
||||
np.linspace(
|
||||
np.log(config.init_resolution),
|
||||
np.log(config.final_resolution),
|
||||
len(config.upsampling_iters) + 1,
|
||||
)
|
||||
)
|
||||
)
|
||||
.astype("int")
|
||||
.tolist()[1:]
|
||||
)
|
||||
super().__init__(config=config, **kwargs)
|
||||
|
||||
def get_training_callbacks(
|
||||
self, training_callback_attributes: TrainingCallbackAttributes
|
||||
) -> List[TrainingCallback]:
|
||||
|
||||
# the callback that we want to run every X iterations after the training iteration
|
||||
def reinitialize_optimizer(
|
||||
self, training_callback_attributes: TrainingCallbackAttributes, step: int # pylint: disable=unused-argument
|
||||
):
|
||||
resolution = self.upsampling_steps.pop(0)
|
||||
|
||||
# upsample the position and direction grids
|
||||
self.field.density_encoding.upsample_grid(resolution)
|
||||
self.field.color_encoding.upsample_grid(resolution)
|
||||
|
||||
# reinitialize the encodings optimizer
|
||||
optimizers_config = training_callback_attributes.optimizers.config
|
||||
enc = training_callback_attributes.pipeline.get_param_groups()["encodings"]
|
||||
lr_init = optimizers_config["encodings"]["optimizer"].lr
|
||||
|
||||
training_callback_attributes.optimizers.optimizers["encodings"] = optimizers_config["encodings"][
|
||||
"optimizer"
|
||||
].setup(params=enc)
|
||||
if optimizers_config["encodings"]["scheduler"]:
|
||||
training_callback_attributes.optimizers.schedulers["encodings"] = optimizers_config["encodings"][
|
||||
"scheduler"
|
||||
].setup(optimizer=training_callback_attributes.optimizers.optimizers["encodings"], lr_init=lr_init)
|
||||
|
||||
callbacks = [
|
||||
TrainingCallback(
|
||||
where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION],
|
||||
iters=self.upsampling_iters,
|
||||
func=reinitialize_optimizer,
|
||||
args=[self, training_callback_attributes],
|
||||
)
|
||||
]
|
||||
return callbacks
|
||||
|
||||
def populate_modules(self):
|
||||
"""Set the fields and modules"""
|
||||
super().populate_modules()
|
||||
|
||||
# setting up fields
|
||||
density_encoding = TensorVMEncoding(
|
||||
resolution=self.init_resolution,
|
||||
num_components=self.num_den_components,
|
||||
)
|
||||
color_encoding = TensorVMEncoding(
|
||||
resolution=self.init_resolution,
|
||||
num_components=self.num_color_components,
|
||||
)
|
||||
|
||||
feature_encoding = NeRFEncoding(in_dim=self.appearance_dim, num_frequencies=2, min_freq_exp=0, max_freq_exp=2)
|
||||
direction_encoding = NeRFEncoding(in_dim=3, num_frequencies=2, min_freq_exp=0, max_freq_exp=2)
|
||||
|
||||
self.field = TensoRFField(
|
||||
self.scene_box.aabb,
|
||||
feature_encoding=feature_encoding,
|
||||
direction_encoding=direction_encoding,
|
||||
density_encoding=density_encoding,
|
||||
color_encoding=color_encoding,
|
||||
appearance_dim=self.appearance_dim,
|
||||
head_mlp_num_layers=2,
|
||||
head_mlp_layer_width=128,
|
||||
use_sh=False,
|
||||
)
|
||||
|
||||
# samplers
|
||||
self.sampler_uniform = UniformSampler(num_samples=self.config.num_samples, single_jitter=True)
|
||||
self.sampler_pdf = PDFSampler(num_samples=self.config.num_samples // 2, single_jitter=True)
|
||||
|
||||
# renderers
|
||||
self.renderer_rgb = RGBRenderer(background_color=colors.WHITE)
|
||||
self.renderer_accumulation = AccumulationRenderer()
|
||||
self.renderer_depth = DepthRenderer()
|
||||
|
||||
# losses
|
||||
self.rgb_loss = MSELoss()
|
||||
|
||||
# metrics
|
||||
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
|
||||
self.ssim = structural_similarity_index_measure
|
||||
self.lpips = LearnedPerceptualImagePatchSimilarity()
|
||||
|
||||
# colliders
|
||||
if self.config.enable_collider:
|
||||
self.collider = AABBBoxCollider(scene_box=self.scene_box)
|
||||
|
||||
def get_param_groups(self) -> Dict[str, List[Parameter]]:
|
||||
param_groups = {}
|
||||
|
||||
param_groups["fields"] = (
|
||||
list(self.field.mlp_head.parameters())
|
||||
+ list(self.field.B.parameters())
|
||||
+ list(self.field.field_output_rgb.parameters())
|
||||
)
|
||||
param_groups["encodings"] = list(self.field.color_encoding.parameters()) + list(
|
||||
self.field.density_encoding.parameters()
|
||||
)
|
||||
|
||||
return param_groups
|
||||
|
||||
def get_outputs(self, ray_bundle: RayBundle):
|
||||
# uniform sampling
|
||||
ray_samples_uniform = self.sampler_uniform(ray_bundle)
|
||||
dens = self.field.get_density(ray_samples_uniform)
|
||||
weights = ray_samples_uniform.get_weights(dens)
|
||||
coarse_accumulation = self.renderer_accumulation(weights)
|
||||
acc_mask = torch.where(coarse_accumulation < 0.0001, False, True).reshape(-1)
|
||||
|
||||
# pdf sampling
|
||||
ray_samples_pdf = self.sampler_pdf(ray_bundle, ray_samples_uniform, weights)
|
||||
|
||||
# fine field:
|
||||
field_outputs_fine = self.field.forward(ray_samples_pdf, acc_mask, colors.WHITE.to(weights.device))
|
||||
|
||||
weights_fine = ray_samples_pdf.get_weights(field_outputs_fine[FieldHeadNames.DENSITY])
|
||||
|
||||
accumulation = self.renderer_accumulation(weights_fine)
|
||||
depth = self.renderer_depth(weights_fine, ray_samples_pdf)
|
||||
|
||||
rgb = self.renderer_rgb(
|
||||
rgb=field_outputs_fine[FieldHeadNames.RGB],
|
||||
weights=weights_fine,
|
||||
)
|
||||
|
||||
rgb = torch.where(accumulation < 0, colors.WHITE.to(rgb.device), rgb)
|
||||
accumulation = torch.clamp(accumulation, min=0)
|
||||
|
||||
outputs = {"rgb": rgb, "accumulation": accumulation, "depth": depth}
|
||||
return outputs
|
||||
|
||||
def get_loss_dict(self, outputs, batch, metrics_dict=None) -> Dict[str, torch.Tensor]:
|
||||
# Scaling metrics by coefficients to create the losses.
|
||||
device = outputs["rgb"].device
|
||||
image = batch["image"].to(device)
|
||||
|
||||
rgb_loss = self.rgb_loss(image, outputs["rgb"])
|
||||
|
||||
loss_dict = {"rgb_loss": rgb_loss}
|
||||
loss_dict = misc.scale_dict(loss_dict, self.config.loss_coefficients)
|
||||
return loss_dict
|
||||
|
||||
def get_image_metrics_and_images(
|
||||
self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor]
|
||||
) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]:
|
||||
image = batch["image"].to(outputs["rgb"].device)
|
||||
rgb = outputs["rgb"]
|
||||
acc = colormaps.apply_colormap(outputs["accumulation"])
|
||||
depth = colormaps.apply_depth_colormap(
|
||||
outputs["depth"],
|
||||
accumulation=outputs["accumulation"],
|
||||
near_plane=self.config.collider_params["near_plane"],
|
||||
far_plane=self.config.collider_params["far_plane"],
|
||||
)
|
||||
|
||||
combined_rgb = torch.cat([image, rgb], dim=1)
|
||||
|
||||
# Switch images from [H, W, C] to [1, C, H, W] for metrics computations
|
||||
image = torch.moveaxis(image, -1, 0)[None, ...]
|
||||
rgb = torch.moveaxis(rgb, -1, 0)[None, ...]
|
||||
|
||||
psnr = self.psnr(image, rgb)
|
||||
ssim = self.ssim(image, rgb)
|
||||
lpips = self.lpips(image, rgb)
|
||||
|
||||
metrics_dict = {
|
||||
"psnr": float(psnr.item()),
|
||||
"ssim": float(ssim.item()),
|
||||
"lpips": float(lpips.item()),
|
||||
}
|
||||
images_dict = {"img": combined_rgb, "accumulation": acc, "depth": depth}
|
||||
return metrics_dict, images_dict
|
||||
Reference in new issue
Block a user