diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index 72e4f20c8..a92c042bf 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -22,6 +22,7 @@ from collections import OrderedDict from typing import Dict import tyro +from nerfstudio.data.pixel_samplers import PairPixelSamplerConfig from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig from nerfstudio.configs.base_config import ViewerConfig @@ -209,6 +210,7 @@ method_configs["depth-nerfacto"] = TrainerConfig( pipeline=VanillaPipelineConfig( datamanager=VanillaDataManagerConfig( _target=VanillaDataManager[DepthDataset], + pixel_sampler=PairPixelSamplerConfig(), dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=4096, eval_num_rays_per_batch=4096, diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index b3f16af1c..e115949d5 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -55,9 +55,9 @@ from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs from nerfstudio.data.dataparsers.blender_dataparser import BlenderDataParserConfig from nerfstudio.data.datasets.base_dataset import InputDataset from nerfstudio.data.pixel_samplers import ( - EquirectangularPixelSampler, - PatchPixelSampler, PixelSampler, + PixelSamplerConfig, + PatchPixelSamplerConfig, ) from nerfstudio.data.utils.dataloaders import ( CacheDataloader, @@ -348,6 +348,8 @@ class VanillaDataManagerConfig(DataManagerConfig): """ patch_size: int = 1 """Size of patch to sample from. If >1, patch-based sampling will be used.""" + pixel_sampler: PixelSamplerConfig = PixelSamplerConfig() + """Specifies the pixel sampler used to sample pixels from images.""" TDataset = TypeVar("TDataset", bound=InputDataset, default=InputDataset) @@ -457,19 +459,18 @@ class VanillaDataManager(DataManager, Generic[TDataset]): scale_factor=self.config.camera_res_scale_factor, ) - def _get_pixel_sampler(self, dataset: TDataset, *args: Any, **kwargs: Any) -> PixelSampler: + def _get_pixel_sampler(self, dataset: TDataset, num_rays_per_batch: int) -> PixelSampler: """Infer pixel sampler to use.""" if self.config.patch_size > 1: - return PatchPixelSampler(*args, **kwargs, patch_size=self.config.patch_size) - - # If all images are equirectangular, use equirectangular pixel sampler - is_equirectangular = dataset.cameras.camera_type == CameraType.EQUIRECTANGULAR.value - if is_equirectangular.all(): - return EquirectangularPixelSampler(*args, **kwargs) - # Otherwise, use the default pixel sampler + return PatchPixelSamplerConfig().setup( + patch_size=self.config.patch_size, num_rays_per_batch=num_rays_per_batch + ) + is_equirectangular = (dataset.cameras.camera_type == CameraType.EQUIRECTANGULAR.value).all() if is_equirectangular.any(): CONSOLE.print("[bold yellow]Warning: Some cameras are equirectangular, but using default pixel sampler.") - return PixelSampler(*args, **kwargs) + return self.config.pixel_sampler.setup( + is_equirectangular=is_equirectangular, num_rays_per_batch=num_rays_per_batch + ) def setup_train(self): """Sets up the data loaders for training""" diff --git a/nerfstudio/data/datasets/depth_dataset.py b/nerfstudio/data/datasets/depth_dataset.py index 1d64cdf1c..85d20ce7c 100644 --- a/nerfstudio/data/datasets/depth_dataset.py +++ b/nerfstudio/data/datasets/depth_dataset.py @@ -18,31 +18,95 @@ Depth dataset. from typing import Dict +import numpy as np + from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs +from nerfstudio.model_components import losses from nerfstudio.data.datasets.base_dataset import InputDataset from nerfstudio.data.utils.data_utils import get_depth_image_from_path +from nerfstudio.utils.rich_utils import CONSOLE + +from typing import Union +from PIL import Image +import torch +from rich.progress import track +from pathlib import Path +import json class DepthDataset(InputDataset): - """Dataset that returns images and depths. + """Dataset that returns images and depths. If no depths are found, then we generate them with Zoe Depth. Args: dataparser_outputs: description of where and how to read input images. scale_factor: The scaling factor for the dataparser outputs. """ - exclude_batch_keys_from_device = InputDataset.exclude_batch_keys_from_device + ["depth_image"] - def __init__(self, dataparser_outputs: DataparserOutputs, scale_factor: float = 1.0): super().__init__(dataparser_outputs, scale_factor) - assert ( - "depth_filenames" in dataparser_outputs.metadata.keys() - and dataparser_outputs.metadata["depth_filenames"] is not None - ) + # if there are no depth images than we want to generate them all with zoe depth + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + if len(dataparser_outputs.image_filenames) > 0 and ( + "depth_filenames" not in dataparser_outputs.metadata.keys() + or dataparser_outputs.metadata["depth_filenames"] is None + ): + losses.DEPTH_METRIC = 0 + CONSOLE.print("[bold yellow] No depth data found!") + cache = dataparser_outputs.image_filenames[0].parent / "depths.npy" + if cache.exists(): + CONSOLE.print("[bold yellow] Loading cache!") + # load all the depths + self.depths = np.load(cache) + self.depths = torch.from_numpy(self.depths).to(device) + else: + depth_tensors = [] + transforms = self._find_transform(dataparser_outputs.image_filenames[0]) + data = dataparser_outputs.image_filenames[0].parent + if transforms is not None: + meta = json.load(open(transforms, "r")) + frames = meta["frames"] + filenames = [data / frames[j]["file_path"].split("/")[-1] for j in range(len(frames))] + else: + meta = None + frames = None + filenames = dataparser_outputs.image_filenames + + repo = "isl-org/ZoeDepth" + self.zoe = torch.compile(torch.hub.load(repo, "ZoeD_NK", pretrained=True).to(device)) + + for i in track(range(len(filenames)), description="Generating depth images"): + image_filename = filenames[i] + pil_image = Image.open(image_filename) + image = np.array(pil_image, dtype="uint8") # shape is (h, w) or (h, w, 3 or 4) + if len(image.shape) == 2: + image = image[:, :, None].repeat(3, axis=2) + image = torch.from_numpy(image.astype("float32") / 255.0) + + with torch.no_grad(): + image = torch.permute(image, (2, 0, 1)).unsqueeze(0).to(device) + if image.shape[1] == 4: + image = image[:, :3, :, :] + depth_tensor = self.zoe.infer(image).squeeze().unsqueeze(-1) + + depth_tensors.append(depth_tensor) + + self.depths = torch.stack(depth_tensors) + np.save(cache, self.depths.cpu().numpy()) + dataparser_outputs.metadata["depth_filenames"] = None + dataparser_outputs.metadata["depth_unit_scale_factor"] = 1.0 + self.metadata["depth_filenames"] = None + self.metadata["depth_unit_scale_factor"] = 1.0 + + else: + losses.DEPTH_METRIC = 0 + self.depth_filenames = self.metadata["depth_filenames"] self.depth_unit_scale_factor = self.metadata["depth_unit_scale_factor"] def get_metadata(self, data: Dict) -> Dict: + if self.depth_filenames is None: + return {"depth_image": self.depths[data["image_idx"]]} + filepath = self.depth_filenames[data["image_idx"]] height = int(self._dataparser_outputs.cameras.height[data["image_idx"]]) width = int(self._dataparser_outputs.cameras.width[data["image_idx"]]) @@ -54,3 +118,11 @@ class DepthDataset(InputDataset): ) return {"depth_image": depth_image} + + def _find_transform(self, image_path: Path) -> Union[Path, None]: + while image_path.parent != image_path: + transform_path = image_path.parent / "transforms.json" + if transform_path.exists(): + return transform_path + image_path = image_path.parent + return None diff --git a/nerfstudio/data/pixel_samplers.py b/nerfstudio/data/pixel_samplers.py index a145dce46..bb5e9d5c5 100644 --- a/nerfstudio/data/pixel_samplers.py +++ b/nerfstudio/data/pixel_samplers.py @@ -17,25 +17,55 @@ Code for sampling pixels. """ import random -from typing import Dict, Optional, Union import torch from jaxtyping import Int from torch import Tensor +from dataclasses import dataclass, field +from typing import ( + Dict, + Optional, + Type, + Union, +) + +from nerfstudio.configs.base_config import ( + InstantiateConfig, +) + + +@dataclass +class PixelSamplerConfig(InstantiateConfig): + """Configuration for pixel sampler instantiation.""" + + _target: Type = field(default_factory=lambda: PixelSampler) + """Target class to instantiate.""" + num_rays_per_batch: int = 4096 + """Number of rays to sample per batch.""" + keep_full_image: bool = False + """Whether or not to include a reference to the full image in returned batch.""" + is_equirectangular: bool = False + """List of whether or not camera i is equirectangular.""" + class PixelSampler: """Samples 'pixel_batch's from 'image_batch's. Args: - num_rays_per_batch: number of rays to sample per batch - keep_full_image: whether or not to include a reference to the full image in returned batch + config: the DataManagerConfig used to instantiate class """ - def __init__(self, num_rays_per_batch: int, keep_full_image: bool = False, **kwargs) -> None: + config: PixelSamplerConfig + + def __init__(self, config: PixelSamplerConfig, **kwargs) -> None: self.kwargs = kwargs - self.num_rays_per_batch = num_rays_per_batch - self.keep_full_image = keep_full_image + self.config = config + # Possibly override some values if they are present in the kwargs dictionary + self.config.num_rays_per_batch = self.kwargs.get("num_rays_per_batch", self.config.num_rays_per_batch) + self.config.keep_full_image = self.kwargs.get("keep_full_image", self.config.keep_full_image) + self.config.is_equirectangular = self.kwargs.get("is_equirectangular", self.config.is_equirectangular) + self.set_num_rays_per_batch(self.config.num_rays_per_batch) def set_num_rays_per_batch(self, num_rays_per_batch: int): """Set the number of rays to sample per batch. @@ -74,6 +104,36 @@ class PixelSampler: return indices + def sample_method_equirectangular( + self, + batch_size: int, + num_images: int, + image_height: int, + image_width: int, + mask: Optional[Tensor] = None, + device: Union[torch.device, str] = "cpu", + ) -> Int[Tensor, "batch_size 3"]: + if isinstance(mask, torch.Tensor): + # Note: if there is a mask, sampling reduces back to uniform sampling, which gives more + # sampling weight to the poles of the image than the equators. + # TODO(kevinddchen): implement the correct mask-sampling method. + + indices = self.sample_method(batch_size, num_images, image_height, image_width, mask=mask, device=device) + else: + # We sample theta uniformly in [0, 2*pi] + # We sample phi in [0, pi] according to the PDF f(phi) = sin(phi) / 2. + # This is done by inverse transform sampling. + # http://corysimon.github.io/articles/uniformdistn-on-sphere/ + num_images_rand = torch.rand(batch_size, device=device) + phi_rand = torch.acos(1 - 2 * torch.rand(batch_size, device=device)) / torch.pi + theta_rand = torch.rand(batch_size, device=device) + indices = torch.floor( + torch.stack((num_images_rand, phi_rand, theta_rand), dim=-1) + * torch.tensor([num_images, image_height, image_width], device=device) + ).long() + + return indices + def collate_image_dataset_batch(self, batch: Dict, num_rays_per_batch: int, keep_full_image: bool = False): """ Operates on a batch of images and samples pixels to use for generating rays. @@ -90,11 +150,21 @@ class PixelSampler: num_images, image_height, image_width, _ = batch["image"].shape if "mask" in batch: - indices = self.sample_method( - num_rays_per_batch, num_images, image_height, image_width, mask=batch["mask"], device=device - ) + if self.config.is_equirectangular: + indices = self.sample_method_equirectangular( + num_rays_per_batch, num_images, image_height, image_width, mask=batch["mask"], device=device + ) + else: + indices = self.sample_method( + num_rays_per_batch, num_images, image_height, image_width, mask=batch["mask"], device=device + ) else: - indices = self.sample_method(num_rays_per_batch, num_images, image_height, image_width, device=device) + if self.config.is_equirectangular: + indices = self.sample_method_equirectangular( + num_rays_per_batch, num_images, image_height, image_width, device=device + ) + else: + indices = self.sample_method(num_rays_per_batch, num_images, image_height, image_width, device=device) c, y, x = (i.flatten() for i in torch.split(indices, 1, dim=-1)) c, y, x = c.cpu(), y.cpu(), x.cpu() @@ -156,7 +226,12 @@ class PixelSampler: image_height, image_width, _ = batch["image"][i].shape if i == num_images - 1: num_rays_in_batch = num_rays_per_batch - (num_images - 1) * num_rays_in_batch - indices = self.sample_method(num_rays_in_batch, 1, image_height, image_width, device=device) + if self.config.is_equirectangular: + indices = self.sample_method_equirectangular( + num_rays_in_batch, 1, image_height, image_width, device=device + ) + else: + indices = self.sample_method(num_rays_in_batch, 1, image_height, image_width, device=device) indices[:, 0] = i all_indices.append(indices) all_images.append(batch["image"][i][indices[:, 1], indices[:, 2]]) @@ -192,56 +267,26 @@ class PixelSampler: if isinstance(image_batch["image"], list): image_batch = dict(image_batch.items()) # copy the dictionary so we don't modify the original pixel_batch = self.collate_image_dataset_batch_list( - image_batch, self.num_rays_per_batch, keep_full_image=self.keep_full_image + image_batch, self.num_rays_per_batch, keep_full_image=self.config.keep_full_image ) elif isinstance(image_batch["image"], torch.Tensor): pixel_batch = self.collate_image_dataset_batch( - image_batch, self.num_rays_per_batch, keep_full_image=self.keep_full_image + image_batch, self.num_rays_per_batch, keep_full_image=self.config.keep_full_image ) else: raise ValueError("image_batch['image'] must be a list or torch.Tensor") return pixel_batch -class EquirectangularPixelSampler(PixelSampler): - """Samples 'pixel_batch's from 'image_batch's. Assumes images are - equirectangular and the sampling is done uniformly on the sphere. +@dataclass +class PatchPixelSamplerConfig(PixelSamplerConfig): + """Config dataclass for PatchPixelSampler.""" - Args: - num_rays_per_batch: number of rays to sample per batch - keep_full_image: whether or not to include a reference to the full image in returned batch - """ - - # overrides base method - def sample_method( - self, - batch_size: int, - num_images: int, - image_height: int, - image_width: int, - mask: Optional[Tensor] = None, - device: Union[torch.device, str] = "cpu", - ) -> Int[Tensor, "batch_size 3"]: - if isinstance(mask, torch.Tensor): - # Note: if there is a mask, sampling reduces back to uniform sampling, which gives more - # sampling weight to the poles of the image than the equators. - # TODO(kevinddchen): implement the correct mask-sampling method. - - indices = super().sample_method(batch_size, num_images, image_height, image_width, mask=mask, device=device) - else: - # We sample theta uniformly in [0, 2*pi] - # We sample phi in [0, pi] according to the PDF f(phi) = sin(phi) / 2. - # This is done by inverse transform sampling. - # http://corysimon.github.io/articles/uniformdistn-on-sphere/ - num_images_rand = torch.rand(batch_size, device=device) - phi_rand = torch.acos(1 - 2 * torch.rand(batch_size, device=device)) / torch.pi - theta_rand = torch.rand(batch_size, device=device) - indices = torch.floor( - torch.stack((num_images_rand, phi_rand, theta_rand), dim=-1) - * torch.tensor([num_images, image_height, image_width], device=device) - ).long() - - return indices + _target: Type = field(default_factory=lambda: PatchPixelSampler) + """Target class to instantiate.""" + patch_size: int = 2 + """Side length of patch. This must be consistent in the method + config in order for samples to be reshaped into patches correctly.""" class PatchPixelSampler(PixelSampler): @@ -249,16 +294,10 @@ class PatchPixelSampler(PixelSampler): from the images randomly. Useful for patch-based losses. Args: - num_rays_per_batch: number of rays to sample per batch - keep_full_image: whether or not to include a reference to the full image in returned batch - patch_size: side length of patch. This must be consistent in the method - config in order for samples to be reshaped into patches correctly. + config: the PatchPixelSamplerConfig used to instantiate class """ - def __init__(self, num_rays_per_batch: int, keep_full_image: bool = False, **kwargs) -> None: - self.patch_size = kwargs["patch_size"] - num_rays = (num_rays_per_batch // (self.patch_size**2)) * (self.patch_size**2) - super().__init__(num_rays, keep_full_image, **kwargs) + config: PatchPixelSamplerConfig def set_num_rays_per_batch(self, num_rays_per_batch: int): """Set the number of rays to sample per batch. Overridden to deal with patch-based sampling. @@ -266,7 +305,7 @@ class PatchPixelSampler(PixelSampler): Args: num_rays_per_batch: number of rays to sample per batch """ - self.num_rays_per_batch = (num_rays_per_batch // (self.patch_size**2)) * (self.patch_size**2) + self.num_rays_per_batch = (num_rays_per_batch // (self.config.patch_size**2)) * (self.config.patch_size**2) # overrides base method def sample_method( @@ -279,19 +318,23 @@ class PatchPixelSampler(PixelSampler): device: Union[torch.device, str] = "cpu", ) -> Int[Tensor, "batch_size 3"]: if isinstance(mask, Tensor): - # Note: if there is a mask, sampling reduces back to uniform sampling - indices = super().sample_method(batch_size, num_images, image_height, image_width, mask=mask, device=device) + # Note: if there is a mask, should switch to the base PixelSampler class + raise NotImplementedError() else: - sub_bs = batch_size // (self.patch_size**2) + sub_bs = batch_size // (self.config.patch_size**2) indices = torch.rand((sub_bs, 3), device=device) * torch.tensor( - [num_images, image_height - self.patch_size, image_width - self.patch_size], + [num_images, image_height - self.config.patch_size, image_width - self.config.patch_size], device=device, ) - indices = indices.view(sub_bs, 1, 1, 3).broadcast_to(sub_bs, self.patch_size, self.patch_size, 3).clone() + indices = ( + indices.view(sub_bs, 1, 1, 3) + .broadcast_to(sub_bs, self.config.patch_size, self.config.patch_size, 3) + .clone() + ) yys, xxs = torch.meshgrid( - torch.arange(self.patch_size, device=device), torch.arange(self.patch_size, device=device) + torch.arange(self.config.patch_size, device=device), torch.arange(self.config.patch_size, device=device) ) indices[:, ..., 1] += yys indices[:, ..., 2] += xxs @@ -300,3 +343,59 @@ class PatchPixelSampler(PixelSampler): indices = indices.flatten(0, 2) return indices + + +@dataclass +class PairPixelSamplerConfig(PixelSamplerConfig): + """Config dataclass for PatchPixelSampler.""" + + _target: Type = field(default_factory=lambda: PairPixelSampler) + """Target class to instantiate.""" + radius: int = 2 + """max distance between pairs of pixels.""" + + +class PairPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods + """Samples pair of pixels from 'image_batch's. Samples pairs of pixels from + from the images randomly within a 'radius' distance apart. Useful for pair-based losses. + + Args: + config: the PairPixelSamplerConfig used to instantiate class + """ + + def __init__(self, config: PairPixelSamplerConfig, **kwargs) -> None: + self.config = config + self.radius = self.config.radius + self.rays_to_sample = self.config.num_rays_per_batch // 2 + super().__init__(self.config, **kwargs) + + # overrides base method + def sample_method( # pylint: disable=no-self-use + self, + batch_size: int, + num_images: int, + image_height: int, + image_width: int, + mask: Optional[Tensor] = None, + device: Union[torch.device, str] = "cpu", + ) -> Int[Tensor, "batch_size 3"]: + if mask: + # Note: if there is a mask, should switch to the base PixelSampler class + + raise NotImplementedError() + else: + s = (self.rays_to_sample, 1) + ns = torch.randint(0, num_images, s, dtype=torch.long, device=device) + hs = torch.randint(self.radius, image_height - self.radius, s, dtype=torch.long, device=device) + ws = torch.randint(self.radius, image_width - self.radius, s, dtype=torch.long, device=device) + indices = torch.concat((ns, hs, ws), dim=1) + + pair_indices = torch.hstack( + ( + torch.zeros(self.rays_to_sample, 1, device=device, dtype=torch.long), + torch.randint(-self.radius, self.radius, (self.rays_to_sample, 2), device=device, dtype=torch.long), + ) + ) + pair_indices += indices + indices = torch.hstack((indices, pair_indices)).view(self.rays_to_sample * 2, 3) + return indices diff --git a/nerfstudio/model_components/losses.py b/nerfstudio/model_components/losses.py index 44fefafd1..8c53c24f1 100644 --- a/nerfstudio/model_components/losses.py +++ b/nerfstudio/model_components/losses.py @@ -35,6 +35,7 @@ EPS = 1.0e-7 # Sigma scale factor from Urban Radiance Fields (Rematas et al., 2022) URF_SIGMA_SCALE_FACTOR = 3.0 +DEPTH_METRIC = 1 class DepthLossType(Enum): @@ -561,3 +562,16 @@ def scale_gradients_by_distance_squared( for key, value in field_outputs.items(): out[key], _ = cast(Tuple[Tensor, Tensor], _GradientScaler.apply(value, scaling)) return out + + +def depth_ranking_loss(rendered_depth, gt_depth): + """ + Depth ranking loss as described in the SparseNeRF paper + Assumes that the layout of the batch comes from a PairPixelSampler, so that adjacent samples in the gt_depth + and rendered_depth are from pixels with a radius of each other + """ + m = 1e-4 + dpt_diff = gt_depth[::2, :] - gt_depth[1::2, :] + out_diff = rendered_depth[::2, :] - rendered_depth[1::2, :] + m + differing_signs = torch.sign(dpt_diff) != torch.sign(out_diff) + return torch.nanmean((out_diff[differing_signs] * torch.sign(out_diff[differing_signs]))) diff --git a/nerfstudio/models/depth_nerfacto.py b/nerfstudio/models/depth_nerfacto.py index e3328fd6d..f3f791e80 100644 --- a/nerfstudio/models/depth_nerfacto.py +++ b/nerfstudio/models/depth_nerfacto.py @@ -22,9 +22,11 @@ from dataclasses import dataclass, field from typing import Dict, Tuple, Type import torch +import numpy as np from nerfstudio.cameras.rays import RayBundle -from nerfstudio.model_components.losses import DepthLossType, depth_loss +from nerfstudio.model_components import losses +from nerfstudio.model_components.losses import DepthLossType, depth_loss, depth_ranking_loss from nerfstudio.models.nerfacto import NerfactoModel, NerfactoModelConfig from nerfstudio.utils import colormaps @@ -98,7 +100,14 @@ class DepthNerfactoModel(NerfactoModel): loss_dict = super().get_loss_dict(outputs, batch, metrics_dict) if self.training: assert metrics_dict is not None and "depth_loss" in metrics_dict - loss_dict["depth_loss"] = self.config.depth_loss_mult * metrics_dict["depth_loss"] + # If real depth used + if losses.DEPTH_METRIC: + loss_dict["depth_loss"] = self.config.depth_loss_mult * metrics_dict["depth_loss"] + else: + loss_dict["depth_ranking"] = np.interp(self.step, [0, 2000], [0, 0.2]) * depth_ranking_loss( + outputs["expected_depth"], batch["depth_image"] + ) + # Else use depth ranking loss return loss_dict diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py index 065fb571e..1122d5c20 100644 --- a/nerfstudio/models/nerfacto.py +++ b/nerfstudio/models/nerfacto.py @@ -220,7 +220,8 @@ class NerfactoModel(Model): # renderers self.renderer_rgb = RGBRenderer(background_color=self.config.background_color) self.renderer_accumulation = AccumulationRenderer() - self.renderer_depth = DepthRenderer() + self.renderer_depth = DepthRenderer(method="median") + self.renderer_expected_depth = DepthRenderer(method="expected") self.renderer_normals = NormalsRenderer() # shaders @@ -233,6 +234,7 @@ class NerfactoModel(Model): self.psnr = PeakSignalNoiseRatio(data_range=1.0) self.ssim = structural_similarity_index_measure self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True) + self.step = 0 def get_param_groups(self) -> Dict[str, List[Parameter]]: param_groups = {} @@ -251,6 +253,7 @@ class NerfactoModel(Model): def set_anneal(step): # https://arxiv.org/pdf/2111.12077.pdf eq. 18 train_frac = np.clip(step / N, 0, 1) + self.step = step def bias(x, b): return b * x / ((b - 1) * x + 1) @@ -286,13 +289,16 @@ class NerfactoModel(Model): ray_samples_list.append(ray_samples) rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights) - depth = self.renderer_depth(weights=weights, ray_samples=ray_samples) + with torch.no_grad(): + depth = self.renderer_depth(weights=weights, ray_samples=ray_samples) + expected_depth = self.renderer_expected_depth(weights=weights, ray_samples=ray_samples) accumulation = self.renderer_accumulation(weights=weights) outputs = { "rgb": rgb, "accumulation": accumulation, "depth": depth, + "expected_depth": expected_depth, } if self.config.predict_normals: diff --git a/pyproject.toml b/pyproject.toml index 804dab015..61e04bc1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,8 @@ dependencies = [ "nuscenes-devkit>=1.1.1", "wandb>=0.13.3", "xatlas", - "trimesh>=3.20.2" + "trimesh>=3.20.2", + "timm==0.6.7" ] [project.urls]