From 33d5052af45c184ca9b5aa231237d9ce9f0a7983 Mon Sep 17 00:00:00 2001 From: terrance Date: Mon, 3 Jul 2023 08:10:13 +0000 Subject: [PATCH] fix mesh export and add new sampling function for random cam datamanager --- .../random_cameras_datamanager.py | 51 +++++++++++++++++-- nerfstudio/exporter/exporter_utils.py | 6 ++- nerfstudio/scripts/exporter.py | 3 +- 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/nerfstudio/data/datamanagers/random_cameras_datamanager.py b/nerfstudio/data/datamanagers/random_cameras_datamanager.py index 4269b428a..975bb0148 100644 --- a/nerfstudio/data/datamanagers/random_cameras_datamanager.py +++ b/nerfstudio/data/datamanagers/random_cameras_datamanager.py @@ -34,7 +34,14 @@ from nerfstudio.configs.config_utils import to_immutable_dict from nerfstudio.data.datamanagers.base_datamanager import DataManager, DataManagerConfig from nerfstudio.data.datasets.base_dataset import InputDataset from nerfstudio.data.scene_box import SceneBox -from nerfstudio.data.utils.dataloaders import RandIndicesEvalDataloader +from nerfstudio.data.utils.dataloaders import RandIndicesEvalDataloader, CacheDataloader +from nerfstudio.data.utils.nerfstudio_collate import nerfstudio_collate +from nerfstudio.data.pixel_samplers import ( + PixelSampler, +) +from nerfstudio.model_components.ray_generators import RayGenerator +from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig + CONSOLE = Console(width=120) @@ -157,7 +164,7 @@ class RandomCamerasDataManagerConfig(DataManagerConfig): _target: Type = field(default_factory=lambda: RandomCamerasDataManager) train_resolution: int = 64 """Training resolution""" - eval_resolution: int = 64 + eval_resolution: int = 256 """Evaluation resolution""" num_eval_angles: int = 256 """Number of evaluation angles""" @@ -232,8 +239,26 @@ class RandomCamerasDataManager(DataManager): # pylint: disable=abstract-method ) self.train_dataset = TrivialDataset(cameras) + + self.train_image_dataloader = CacheDataloader( + self.train_dataset, + num_images_to_sample_from=256, + num_times_to_repeat_images=-1, + device=self.device, + num_workers=self.world_size * 4, + pin_memory=True, + collate_fn=nerfstudio_collate, + exclude_batch_keys_from_device=self.train_dataset.exclude_batch_keys_from_device, + ) + + self.iter_train_image_dataloader = iter(self.train_image_dataloader) + + self.train_pixel_sampler = PixelSampler(num_rays_per_batch=self.get_eval_rays_per_batch()) + self.eval_dataset = TrivialDataset(cameras) + self.horizontal_rotation_warmup = self.config.horizontal_rotation_warmup + self.eval_dataloader = RandIndicesEvalDataloader( input_dataset=self.eval_dataset, device=self.device, @@ -247,7 +272,7 @@ class RandomCamerasDataManager(DataManager): # pylint: disable=abstract-method """Returns the next batch of data from the train dataloader.""" self.train_count += 1 - horizontal_range = min((step / max(1, self.config.horizontal_rotation_warmup)), 1) * 180 + horizontal_range = min((step / max(1, self.horizontal_rotation_warmup)), 1) * 180 cameras, vertical_rotation, central_rotation = random_train_pose( self.config.train_images_per_batch, @@ -269,6 +294,26 @@ class RandomCamerasDataManager(DataManager): # pylint: disable=abstract-method "initialization": True, } + def random_train(self, step: int) -> Tuple[RayBundle, Dict]: + """Returns a batch of randomly sampled camera rays. For use during point cloud generation""" + + assert self.train_dataset is not None + temp_camera_optimizer = CameraOptimizerConfig().setup( + num_cameras=self.train_dataset.cameras.size, device=self.device + ) + self.train_ray_generator = RayGenerator( + self.train_dataset.cameras.to(self.device), + temp_camera_optimizer, + ) + + image_batch = next(self.iter_train_image_dataloader) + assert image_batch is not None + batch = self.train_pixel_sampler.sample(image_batch) + ray_indices = batch["indices"] + + ray_bundle = self.train_ray_generator(ray_indices) + return ray_bundle, batch + def next_eval(self, step: int) -> Tuple[RayBundle, Dict]: """Returns the next batch of data from the eval dataloader.""" self.eval_count += 1 diff --git a/nerfstudio/exporter/exporter_utils.py b/nerfstudio/exporter/exporter_utils.py index 68211b5c3..eeaf9b6a3 100644 --- a/nerfstudio/exporter/exporter_utils.py +++ b/nerfstudio/exporter/exporter_utils.py @@ -41,6 +41,7 @@ from nerfstudio.cameras.cameras import Cameras from nerfstudio.data.datasets.base_dataset import InputDataset from nerfstudio.pipelines.base_pipeline import Pipeline, VanillaPipeline from nerfstudio.utils.rich_utils import CONSOLE, ItersPerSecColumn +from nerfstudio.data.datamanagers.random_cameras_datamanager import RandomCamerasDataManager @dataclass @@ -128,7 +129,10 @@ def generate_point_cloud( normal = None with torch.no_grad(): - ray_bundle, _ = pipeline.datamanager.next_train(0) + if isinstance(pipeline.datamanager, RandomCamerasDataManager): + ray_bundle, _ = pipeline.datamanager.random_train(0) + else: + ray_bundle, _ = pipeline.datamanager.next_train(0) outputs = pipeline.model(ray_bundle) if rgb_output_name not in outputs: CONSOLE.rule("Error", style="red") diff --git a/nerfstudio/scripts/exporter.py b/nerfstudio/scripts/exporter.py index 153ab4e4b..4e120faed 100644 --- a/nerfstudio/scripts/exporter.py +++ b/nerfstudio/scripts/exporter.py @@ -34,6 +34,7 @@ from typing_extensions import Annotated, Literal from nerfstudio.cameras.rays import RayBundle from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager +from nerfstudio.data.datamanagers.random_cameras_datamanager import RandomCamerasDataManager from nerfstudio.exporter import texture_utils, tsdf_utils from nerfstudio.exporter.exporter_utils import ( collect_camera_poses, @@ -285,7 +286,7 @@ class ExportPoissonMesh(Exporter): validate_pipeline(self.normal_method, self.normal_output_name, pipeline) # Increase the batchsize to speed up the evaluation. - assert isinstance(pipeline.datamanager, VanillaDataManager) + assert isinstance(pipeline.datamanager, (VanillaDataManager, RandomCamerasDataManager)) assert pipeline.datamanager.train_pixel_sampler is not None pipeline.datamanager.train_pixel_sampler.num_rays_per_batch = self.num_rays_per_batch