mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-18 13:53:44 +08:00
Hard coding less, offloading to config
This commit is contained in:
1 parent
22340b37f3
commit
e2b345ed8c
3 files changed
+44
-25
No files matched your search
@@ -160,6 +160,16 @@ class DreamFusionDataManagerConfig(VanillaDataManagerConfig):
|
||||
"""Number of images per batch for training"""
|
||||
eval_images_per_batch: int = 1
|
||||
"""Number of images per batch for evaluation"""
|
||||
radius_mean: float = 1.6
|
||||
"""Mean radius of camera orbit"""
|
||||
radius_std: float = 0.2
|
||||
"""Std of radius of camera orbit"""
|
||||
focal_range: Tuple[float, float] = (0.75, 1.35)
|
||||
"""Range of focal length"""
|
||||
vertical_rotation_range: Tuple[float, float] = (-90, 30)
|
||||
"""Range of vertical rotation"""
|
||||
jitter_std: float = 0.05
|
||||
"""Std of camera direction jitter, so we don't just point the cameras towards the center every time"""
|
||||
|
||||
|
||||
class DreamFusionDataManager(VanillaDataManager): # pylint: disable=abstract-method
|
||||
@@ -196,19 +206,19 @@ class DreamFusionDataManager(VanillaDataManager): # pylint: disable=abstract-me
|
||||
self.test_split = "test" if test_mode in ["test", "inference"] else "val"
|
||||
self.dataparser = self.config.dataparser.setup()
|
||||
|
||||
self.eval_cameras, _, _ = random_train_pose(
|
||||
cameras, vertical_rotation, central_rotation = random_train_pose(
|
||||
self.config.num_eval_angles,
|
||||
self.config.eval_resolution,
|
||||
device=self.device,
|
||||
radius_mean=1.5,
|
||||
radius_std=0,
|
||||
focal_range=[1, 1],
|
||||
vertical_rotation_range=[-30, 30],
|
||||
# jitter_std=0,
|
||||
radius_mean=self.config.radius_mean,
|
||||
radius_std=self.config.radius_std,
|
||||
focal_range=self.config.focal_range,
|
||||
vertical_rotation_range=self.config.vertical_rotation_range,
|
||||
jitter_std=self.config.jitter_std,
|
||||
)
|
||||
|
||||
self.train_dataset = TrivialDataset(self.eval_cameras)
|
||||
self.eval_dataset = TrivialDataset(self.eval_cameras)
|
||||
self.train_dataset = TrivialDataset(cameras)
|
||||
self.eval_dataset = TrivialDataset(cameras)
|
||||
|
||||
# pylint: disable=non-parent-init-called
|
||||
DataManager.__init__(self)
|
||||
@@ -234,11 +244,11 @@ class DreamFusionDataManager(VanillaDataManager): # pylint: disable=abstract-me
|
||||
self.config.train_images_per_batch,
|
||||
self.config.train_resolution,
|
||||
device=self.device,
|
||||
radius_mean=1.5,
|
||||
radius_std=0,
|
||||
focal_range=[1, 1],
|
||||
# vertical_rotation_range=[-180, 180],
|
||||
# jitter_std=0,
|
||||
radius_mean=self.config.radius_mean,
|
||||
radius_std=self.config.radius_std,
|
||||
focal_range=self.config.focal_range,
|
||||
vertical_rotation_range=self.config.vertical_rotation_range,
|
||||
jitter_std=self.config.jitter_std,
|
||||
)
|
||||
ray_bundle = cameras.generate_rays(torch.tensor(list(range(self.config.train_images_per_batch)))).flatten()
|
||||
|
||||
@@ -263,11 +273,11 @@ class DreamFusionDataManager(VanillaDataManager): # pylint: disable=abstract-me
|
||||
self.config.eval_images_per_batch,
|
||||
self.config.eval_resolution,
|
||||
device=self.device,
|
||||
radius_mean=1.5,
|
||||
radius_std=0,
|
||||
focal_range=[1, 1],
|
||||
vertical_rotation_range=[-180, 180],
|
||||
jitter_std=0,
|
||||
radius_mean=self.config.radius_mean,
|
||||
radius_std=self.config.radius_std,
|
||||
focal_range=self.config.focal_range,
|
||||
vertical_rotation_range=self.config.vertical_rotation_range,
|
||||
jitter_std=self.config.jitter_std,
|
||||
)
|
||||
ray_bundle = cameras.generate_rays(torch.tensor([[i] for i in range(self.config.train_images_per_batch)]))
|
||||
|
||||
|
||||
@@ -105,8 +105,7 @@ class StableDiffusion(nn.Module):
|
||||
return text_embeddings
|
||||
|
||||
def sds_loss(self, text_embeddings, nerf_output, guidance_scale=100):
|
||||
|
||||
nerf_output_np = nerf_output.view(3, 64, 64).permute(1, 2, 0).detach().cpu().numpy()
|
||||
nerf_output_np = nerf_output.squeeze(0).permute(1, 2, 0).detach().cpu().numpy()
|
||||
nerf_output_np = np.clip(nerf_output_np, 0.0, 1.0)
|
||||
plt.imsave("nerf_output.png", nerf_output_np)
|
||||
nerf_output = F.interpolate(nerf_output, (IMG_DIM, IMG_DIM), mode="bilinear")
|
||||
@@ -215,7 +214,7 @@ class StableDiffusion(nn.Module):
|
||||
latents=latents,
|
||||
num_inference_steps=num_inference_steps,
|
||||
guidance_scale=guidance_scale,
|
||||
) # [1, 4, 64, 64]
|
||||
) # [1, 4, resolution, resolution]
|
||||
|
||||
diffused_img = self.latents_to_img(latents)
|
||||
diffused_img = diffused_img.detach().cpu().permute(0, 2, 3, 1).numpy()
|
||||
|
||||
@@ -98,17 +98,27 @@ class DreamfusionPipeline(VanillaPipeline):
|
||||
metrics_dict = self.model.get_metrics_dict(model_outputs, batch)
|
||||
loss_dict = self.model.get_loss_dict(model_outputs, batch, metrics_dict)
|
||||
|
||||
train_output = model_outputs["train_output"].view(1, 64, 64, 3).permute(0, 3, 1, 2)
|
||||
train_output = (
|
||||
model_outputs["train_output"]
|
||||
.view(1, self.config.datamanager.train_resolution, self.config.datamanager.train_resolution, 3)
|
||||
.permute(0, 3, 1, 2)
|
||||
)
|
||||
|
||||
accumulation = model_outputs["accumulation"].view(64, 64).detach().cpu().numpy()
|
||||
accumulation = (
|
||||
model_outputs["accumulation"]
|
||||
.view(self.config.datamanager.train_resolution, self.config.datamanager.train_resolution)
|
||||
.detach()
|
||||
.cpu()
|
||||
.numpy()
|
||||
)
|
||||
accumulation = np.clip(accumulation, 0.0, 1.0)
|
||||
plt.imsave("nerf_accumulation.jpg", accumulation)
|
||||
|
||||
# background = model_outputs["background"].view(64, 64, 3).detach().cpu().numpy()
|
||||
# background = model_outputs["background"].view(res, res, 3).detach().cpu().numpy()
|
||||
# background = np.clip(background, 0.0, 1.0)
|
||||
# plt.imsave("nerf_background.jpg", background)
|
||||
|
||||
# shaded = model_outputs["shaded"].view(64, 64, 3).detach().cpu().numpy()
|
||||
# shaded = model_outputs["shaded"].view(res, res, 3).detach().cpu().numpy()
|
||||
# shaded = np.clip(shaded, 0.0, 1.0)
|
||||
# plt.imsave("nerf_textureless.jpg", shaded)
|
||||
|
||||
|
||||
Reference in new issue
Block a user