Hard coding less, offloading to config

This commit is contained in:
Jake Austin committed 2022-12-31 03:26:08 +00:00
1 parent 22340b37f3
commit e2b345ed8c
3 files changed
+44 -25

No files matched your search

@@ -160,6 +160,16 @@ class DreamFusionDataManagerConfig(VanillaDataManagerConfig):
"""Number of images per batch for training"""
eval_images_per_batch: int = 1
"""Number of images per batch for evaluation"""
radius_mean: float = 1.6
"""Mean radius of camera orbit"""
radius_std: float = 0.2
"""Std of radius of camera orbit"""
focal_range: Tuple[float, float] = (0.75, 1.35)
"""Range of focal length"""
vertical_rotation_range: Tuple[float, float] = (-90, 30)
"""Range of vertical rotation"""
jitter_std: float = 0.05
"""Std of camera direction jitter, so we don't just point the cameras towards the center every time"""
class DreamFusionDataManager(VanillaDataManager): # pylint: disable=abstract-method
@@ -196,19 +206,19 @@ class DreamFusionDataManager(VanillaDataManager): # pylint: disable=abstract-me
self.test_split = "test" if test_mode in ["test", "inference"] else "val"
self.dataparser = self.config.dataparser.setup()
self.eval_cameras, _, _ = random_train_pose(
cameras, vertical_rotation, central_rotation = random_train_pose(
self.config.num_eval_angles,
self.config.eval_resolution,
device=self.device,
radius_mean=1.5,
radius_std=0,
focal_range=[1, 1],
vertical_rotation_range=[-30, 30],
# jitter_std=0,
radius_mean=self.config.radius_mean,
radius_std=self.config.radius_std,
focal_range=self.config.focal_range,
vertical_rotation_range=self.config.vertical_rotation_range,
jitter_std=self.config.jitter_std,
)
self.train_dataset = TrivialDataset(self.eval_cameras)
self.eval_dataset = TrivialDataset(self.eval_cameras)
self.train_dataset = TrivialDataset(cameras)
self.eval_dataset = TrivialDataset(cameras)
# pylint: disable=non-parent-init-called
DataManager.__init__(self)
@@ -234,11 +244,11 @@ class DreamFusionDataManager(VanillaDataManager): # pylint: disable=abstract-me
self.config.train_images_per_batch,
self.config.train_resolution,
device=self.device,
radius_mean=1.5,
radius_std=0,
focal_range=[1, 1],
# vertical_rotation_range=[-180, 180],
# jitter_std=0,
radius_mean=self.config.radius_mean,
radius_std=self.config.radius_std,
focal_range=self.config.focal_range,
vertical_rotation_range=self.config.vertical_rotation_range,
jitter_std=self.config.jitter_std,
)
ray_bundle = cameras.generate_rays(torch.tensor(list(range(self.config.train_images_per_batch)))).flatten()
@@ -263,11 +273,11 @@ class DreamFusionDataManager(VanillaDataManager): # pylint: disable=abstract-me
self.config.eval_images_per_batch,
self.config.eval_resolution,
device=self.device,
radius_mean=1.5,
radius_std=0,
focal_range=[1, 1],
vertical_rotation_range=[-180, 180],
jitter_std=0,
radius_mean=self.config.radius_mean,
radius_std=self.config.radius_std,
focal_range=self.config.focal_range,
vertical_rotation_range=self.config.vertical_rotation_range,
jitter_std=self.config.jitter_std,
)
ray_bundle = cameras.generate_rays(torch.tensor([[i] for i in range(self.config.train_images_per_batch)]))
+2 -3
View File
@@ -105,8 +105,7 @@ class StableDiffusion(nn.Module):
return text_embeddings
def sds_loss(self, text_embeddings, nerf_output, guidance_scale=100):
nerf_output_np = nerf_output.view(3, 64, 64).permute(1, 2, 0).detach().cpu().numpy()
nerf_output_np = nerf_output.squeeze(0).permute(1, 2, 0).detach().cpu().numpy()
nerf_output_np = np.clip(nerf_output_np, 0.0, 1.0)
plt.imsave("nerf_output.png", nerf_output_np)
nerf_output = F.interpolate(nerf_output, (IMG_DIM, IMG_DIM), mode="bilinear")
@@ -215,7 +214,7 @@ class StableDiffusion(nn.Module):
latents=latents,
num_inference_steps=num_inference_steps,
guidance_scale=guidance_scale,
) # [1, 4, 64, 64]
) # [1, 4, resolution, resolution]
diffused_img = self.latents_to_img(latents)
diffused_img = diffused_img.detach().cpu().permute(0, 2, 3, 1).numpy()
+14 -4
View File
@@ -98,17 +98,27 @@ class DreamfusionPipeline(VanillaPipeline):
metrics_dict = self.model.get_metrics_dict(model_outputs, batch)
loss_dict = self.model.get_loss_dict(model_outputs, batch, metrics_dict)
train_output = model_outputs["train_output"].view(1, 64, 64, 3).permute(0, 3, 1, 2)
train_output = (
model_outputs["train_output"]
.view(1, self.config.datamanager.train_resolution, self.config.datamanager.train_resolution, 3)
.permute(0, 3, 1, 2)
)
accumulation = model_outputs["accumulation"].view(64, 64).detach().cpu().numpy()
accumulation = (
model_outputs["accumulation"]
.view(self.config.datamanager.train_resolution, self.config.datamanager.train_resolution)
.detach()
.cpu()
.numpy()
)
accumulation = np.clip(accumulation, 0.0, 1.0)
plt.imsave("nerf_accumulation.jpg", accumulation)
# background = model_outputs["background"].view(64, 64, 3).detach().cpu().numpy()
# background = model_outputs["background"].view(res, res, 3).detach().cpu().numpy()
# background = np.clip(background, 0.0, 1.0)
# plt.imsave("nerf_background.jpg", background)
# shaded = model_outputs["shaded"].view(64, 64, 3).detach().cpu().numpy()
# shaded = model_outputs["shaded"].view(res, res, 3).detach().cpu().numpy()
# shaded = np.clip(shaded, 0.0, 1.0)
# plt.imsave("nerf_textureless.jpg", shaded)