From e2b345ed8c837551de185856d1608f3e4c39c6c4 Mon Sep 17 00:00:00 2001 From: Jake Austin Date: Sat, 31 Dec 2022 03:26:08 +0000 Subject: [PATCH] Hard coding less, offloading to config --- .../datamanagers/dreamfusion_datamanager.py | 46 +++++++++++-------- nerfstudio/generative/stable_diffusion.py | 5 +- nerfstudio/pipelines/dreamfusion_pipeline.py | 18 ++++++-- 3 files changed, 44 insertions(+), 25 deletions(-) diff --git a/nerfstudio/data/datamanagers/dreamfusion_datamanager.py b/nerfstudio/data/datamanagers/dreamfusion_datamanager.py index e7f978ebe..454c5880d 100644 --- a/nerfstudio/data/datamanagers/dreamfusion_datamanager.py +++ b/nerfstudio/data/datamanagers/dreamfusion_datamanager.py @@ -160,6 +160,16 @@ class DreamFusionDataManagerConfig(VanillaDataManagerConfig): """Number of images per batch for training""" eval_images_per_batch: int = 1 """Number of images per batch for evaluation""" + radius_mean: float = 1.6 + """Mean radius of camera orbit""" + radius_std: float = 0.2 + """Std of radius of camera orbit""" + focal_range: Tuple[float, float] = (0.75, 1.35) + """Range of focal length""" + vertical_rotation_range: Tuple[float, float] = (-90, 30) + """Range of vertical rotation""" + jitter_std: float = 0.05 + """Std of camera direction jitter, so we don't just point the cameras towards the center every time""" class DreamFusionDataManager(VanillaDataManager): # pylint: disable=abstract-method @@ -196,19 +206,19 @@ class DreamFusionDataManager(VanillaDataManager): # pylint: disable=abstract-me self.test_split = "test" if test_mode in ["test", "inference"] else "val" self.dataparser = self.config.dataparser.setup() - self.eval_cameras, _, _ = random_train_pose( + cameras, vertical_rotation, central_rotation = random_train_pose( self.config.num_eval_angles, self.config.eval_resolution, device=self.device, - radius_mean=1.5, - radius_std=0, - focal_range=[1, 1], - vertical_rotation_range=[-30, 30], - # jitter_std=0, + radius_mean=self.config.radius_mean, + radius_std=self.config.radius_std, + focal_range=self.config.focal_range, + vertical_rotation_range=self.config.vertical_rotation_range, + jitter_std=self.config.jitter_std, ) - self.train_dataset = TrivialDataset(self.eval_cameras) - self.eval_dataset = TrivialDataset(self.eval_cameras) + self.train_dataset = TrivialDataset(cameras) + self.eval_dataset = TrivialDataset(cameras) # pylint: disable=non-parent-init-called DataManager.__init__(self) @@ -234,11 +244,11 @@ class DreamFusionDataManager(VanillaDataManager): # pylint: disable=abstract-me self.config.train_images_per_batch, self.config.train_resolution, device=self.device, - radius_mean=1.5, - radius_std=0, - focal_range=[1, 1], - # vertical_rotation_range=[-180, 180], - # jitter_std=0, + radius_mean=self.config.radius_mean, + radius_std=self.config.radius_std, + focal_range=self.config.focal_range, + vertical_rotation_range=self.config.vertical_rotation_range, + jitter_std=self.config.jitter_std, ) ray_bundle = cameras.generate_rays(torch.tensor(list(range(self.config.train_images_per_batch)))).flatten() @@ -263,11 +273,11 @@ class DreamFusionDataManager(VanillaDataManager): # pylint: disable=abstract-me self.config.eval_images_per_batch, self.config.eval_resolution, device=self.device, - radius_mean=1.5, - radius_std=0, - focal_range=[1, 1], - vertical_rotation_range=[-180, 180], - jitter_std=0, + radius_mean=self.config.radius_mean, + radius_std=self.config.radius_std, + focal_range=self.config.focal_range, + vertical_rotation_range=self.config.vertical_rotation_range, + jitter_std=self.config.jitter_std, ) ray_bundle = cameras.generate_rays(torch.tensor([[i] for i in range(self.config.train_images_per_batch)])) diff --git a/nerfstudio/generative/stable_diffusion.py b/nerfstudio/generative/stable_diffusion.py index 5f12704a8..5dee76ceb 100644 --- a/nerfstudio/generative/stable_diffusion.py +++ b/nerfstudio/generative/stable_diffusion.py @@ -105,8 +105,7 @@ class StableDiffusion(nn.Module): return text_embeddings def sds_loss(self, text_embeddings, nerf_output, guidance_scale=100): - - nerf_output_np = nerf_output.view(3, 64, 64).permute(1, 2, 0).detach().cpu().numpy() + nerf_output_np = nerf_output.squeeze(0).permute(1, 2, 0).detach().cpu().numpy() nerf_output_np = np.clip(nerf_output_np, 0.0, 1.0) plt.imsave("nerf_output.png", nerf_output_np) nerf_output = F.interpolate(nerf_output, (IMG_DIM, IMG_DIM), mode="bilinear") @@ -215,7 +214,7 @@ class StableDiffusion(nn.Module): latents=latents, num_inference_steps=num_inference_steps, guidance_scale=guidance_scale, - ) # [1, 4, 64, 64] + ) # [1, 4, resolution, resolution] diffused_img = self.latents_to_img(latents) diffused_img = diffused_img.detach().cpu().permute(0, 2, 3, 1).numpy() diff --git a/nerfstudio/pipelines/dreamfusion_pipeline.py b/nerfstudio/pipelines/dreamfusion_pipeline.py index 84030af28..667ac7072 100644 --- a/nerfstudio/pipelines/dreamfusion_pipeline.py +++ b/nerfstudio/pipelines/dreamfusion_pipeline.py @@ -98,17 +98,27 @@ class DreamfusionPipeline(VanillaPipeline): metrics_dict = self.model.get_metrics_dict(model_outputs, batch) loss_dict = self.model.get_loss_dict(model_outputs, batch, metrics_dict) - train_output = model_outputs["train_output"].view(1, 64, 64, 3).permute(0, 3, 1, 2) + train_output = ( + model_outputs["train_output"] + .view(1, self.config.datamanager.train_resolution, self.config.datamanager.train_resolution, 3) + .permute(0, 3, 1, 2) + ) - accumulation = model_outputs["accumulation"].view(64, 64).detach().cpu().numpy() + accumulation = ( + model_outputs["accumulation"] + .view(self.config.datamanager.train_resolution, self.config.datamanager.train_resolution) + .detach() + .cpu() + .numpy() + ) accumulation = np.clip(accumulation, 0.0, 1.0) plt.imsave("nerf_accumulation.jpg", accumulation) - # background = model_outputs["background"].view(64, 64, 3).detach().cpu().numpy() + # background = model_outputs["background"].view(res, res, 3).detach().cpu().numpy() # background = np.clip(background, 0.0, 1.0) # plt.imsave("nerf_background.jpg", background) - # shaded = model_outputs["shaded"].view(64, 64, 3).detach().cpu().numpy() + # shaded = model_outputs["shaded"].view(res, res, 3).detach().cpu().numpy() # shaded = np.clip(shaded, 0.0, 1.0) # plt.imsave("nerf_textureless.jpg", shaded)