fixed viewer with dream embedding

This commit is contained in:
terrancewang committed 2023-03-07 15:56:49 -08:00
1 parent e007d84edd
commit 5a2985c5f5
3 files changed
+188 -78

No files matched your search

@@ -54,7 +54,12 @@ class TrivialDataset(InputDataset):
return self.size
def __getitem__(self, index: int) -> Dict:
return {"image": torch.cat([torch.ones(128, 256, 3), torch.zeros(128, 256, 3)], dim=0), "image_idx": index}
return {
"image": torch.cat(
[torch.ones(128, 256, 3), torch.zeros(128, 256, 3)], dim=0
),
"image_idx": index,
}
def random_train_pose(
@@ -84,14 +89,19 @@ def random_train_pose(
poses: [size, 4, 4]
"""
vertical_rotation_range = (vertical_rotation_range[0] + 90, vertical_rotation_range[1] + 90)
vertical_rotation_range = (
vertical_rotation_range[0] + 90,
vertical_rotation_range[1] + 90,
)
# This is the uniform sample on the part of the sphere we care about where 0 = 0 degrees and 1 = 360 degrees
sampled_uniform = (
torch.rand(size) * (vertical_rotation_range[1] - vertical_rotation_range[0]) + vertical_rotation_range[0]
torch.rand(size) * (vertical_rotation_range[1] - vertical_rotation_range[0])
+ vertical_rotation_range[0]
) / 180
vertical_rotation = torch.arccos(1 - 2 * sampled_uniform)
central_rotation = torch.deg2rad(
torch.rand(size) * (central_rotation_range[1] - central_rotation_range[0]) + central_rotation_range[0]
torch.rand(size) * (central_rotation_range[1] - central_rotation_range[0])
+ central_rotation_range[0]
)
c_cos = torch.cos(central_rotation)
@@ -121,7 +131,9 @@ def random_train_pose(
# Default directions are facing in the -z direction, so origins should face opposite way
origins = torch.stack([torch.tensor([0, 0, 1])] * size, dim=0)
origins = (origins * radius_mean) + (origins * (torch.randn((origins.shape)) * radius_std))
origins = (origins * radius_mean) + (
origins * (torch.randn((origins.shape)) * radius_std)
)
R = torch.bmm(rot_z, rot_x) # Want to have Rx @ Ry @ origin
t = (
torch.bmm(R, origins.unsqueeze(-1))
@@ -158,7 +170,7 @@ class DreamFusionDataManagerConfig(DataManagerConfig):
"""Number of images per batch for training"""
eval_images_per_batch: int = 1
"""Number of images per batch for evaluation"""
radius_mean: float = 3.0
radius_mean: float = 2.5
"""Mean radius of camera orbit"""
radius_std: float = 0.1
"""Std of radius of camera orbit"""
@@ -208,7 +220,9 @@ class DreamFusionDataManager(DataManager): # pylint: disable=abstract-method
self.test_split = "test" if test_mode in ["test", "inference"] else "val"
if self.config.data is not None:
CONSOLE.print("[red] --data should not be used with the DreamFusionDataManager[/red]")
CONSOLE.print(
"[red] --data should not be used with the DreamFusionDataManager[/red]"
)
sys.exit(1)
cameras, _, _ = random_train_pose(
@@ -265,7 +279,9 @@ class DreamFusionDataManager(DataManager): # pylint: disable=abstract-method
center=self.config.center,
central_rotation_range=(-horizontal_range, horizontal_range),
)
ray_bundle = cameras.generate_rays(torch.tensor(list(range(self.config.train_images_per_batch)))).flatten()
ray_bundle = cameras.generate_rays(
torch.tensor(list(range(self.config.train_images_per_batch)))
).flatten()
# camera_idx = torch.randint(0, self.eval_cameras.shape[0], [1], dtype=torch.long, device=self.device)
# ray_bundle = self.eval_cameras.generate_rays(camera_idx).flatten()
@@ -310,7 +326,9 @@ class DreamFusionDataManager(DataManager): # pylint: disable=abstract-method
def get_eval_rays_per_batch(self) -> int:
return self.config.eval_resolution**2
def get_param_groups(self) -> Dict[str, List[Parameter]]: # pylint: disable=no-self-use
def get_param_groups(
self,
) -> Dict[str, List[Parameter]]: # pylint: disable=no-self-use
"""Get the param groups for the data manager.
Returns:
A list of dictionaries containing the data manager's param groups.
+26 -9
View File
@@ -44,9 +44,13 @@ class ModelConfig(InstantiateConfig):
"""target class to instantiate"""
enable_collider: bool = True
"""Whether to create a scene collider to filter rays."""
collider_params: Optional[Dict[str, float]] = to_immutable_dict({"near_plane": 2.0, "far_plane": 6.0})
collider_params: Optional[Dict[str, float]] = to_immutable_dict(
{"near_plane": 2.0, "far_plane": 6.0}
)
"""parameters to instantiate scene collider with"""
loss_coefficients: Dict[str, float] = to_immutable_dict({"rgb_loss_coarse": 1.0, "rgb_loss_fine": 1.0})
loss_coefficients: Dict[str, float] = to_immutable_dict(
{"rgb_loss_coarse": 1.0, "rgb_loss_fine": 1.0}
)
"""parameters to instantiate density field with"""
eval_num_rays_per_chunk: int = 4096
"""specifies number of rays per chunk during eval"""
@@ -76,7 +80,9 @@ class Model(nn.Module):
super().__init__()
self.config = config
self.scene_box = scene_box
self.render_aabb = None # the box that we want to render - should be a subset of scene_box
self.render_aabb = (
None # the box that we want to render - should be a subset of scene_box
)
self.num_train_data = num_train_data
self.kwargs = kwargs
self.collider = None
@@ -92,7 +98,8 @@ class Model(nn.Module):
return self.device_indicator_param.device
def get_training_callbacks( # pylint:disable=no-self-use
self, training_callback_attributes: TrainingCallbackAttributes # pylint: disable=unused-argument
self,
training_callback_attributes: TrainingCallbackAttributes, # pylint: disable=unused-argument
) -> List[TrainingCallback]:
"""Returns a list of callbacks that run functions at the specified training iterations."""
return []
@@ -104,7 +111,8 @@ class Model(nn.Module):
if self.config.enable_collider:
self.collider = NearFarCollider(
near_plane=self.config.collider_params["near_plane"], far_plane=self.config.collider_params["far_plane"]
near_plane=self.config.collider_params["near_plane"],
far_plane=self.config.collider_params["far_plane"],
)
@abstractmethod
@@ -152,7 +160,9 @@ class Model(nn.Module):
return {}
@abstractmethod
def get_loss_dict(self, outputs, batch, metrics_dict=None) -> Dict[str, torch.Tensor]:
def get_loss_dict(
self, outputs, batch, metrics_dict=None
) -> Dict[str, torch.Tensor]:
"""Computes and returns the losses dict.
Args:
@@ -162,7 +172,9 @@ class Model(nn.Module):
"""
@torch.no_grad()
def get_outputs_for_camera_ray_bundle(self, camera_ray_bundle: RayBundle) -> Dict[str, torch.Tensor]:
def get_outputs_for_camera_ray_bundle(
self, camera_ray_bundle: RayBundle
) -> Dict[str, torch.Tensor]:
"""Takes in camera parameters and computes the output of the model.
Args:
@@ -175,7 +187,9 @@ class Model(nn.Module):
for i in range(0, num_rays, num_rays_per_chunk):
start_idx = i
end_idx = i + num_rays_per_chunk
ray_bundle = camera_ray_bundle.get_row_major_sliced_ray_bundle(start_idx, end_idx)
ray_bundle = camera_ray_bundle.get_row_major_sliced_ray_bundle(
start_idx, end_idx
)
outputs = self.forward(ray_bundle=ray_bundle)
for output_name, output in outputs.items(): # type: ignore
outputs_lists[output_name].append(output)
@@ -210,7 +224,10 @@ class Model(nn.Module):
Args:
loaded_state: dictionary of pre-trained model states
"""
state = {key.replace("module.", ""): value for key, value in loaded_state["model"].items()}
state = {
key.replace("module.", ""): value
for key, value in loaded_state["model"].items()
}
self.load_state_dict(state) # type: ignore
def update_to_step(self, step: int) -> None:
+135 -60
View File
@@ -1,14 +1,12 @@
from __future__ import annotations
from collections import defaultdict
from dataclasses import dataclass, field
from typing import Dict, List, Optional, Tuple, Type
import mediapy
import numpy as np
import torch
from torch.nn import Parameter
from typing_extensions import Literal
from nerfstudio.cameras.rays import RayBundle
@@ -27,14 +25,9 @@ from nerfstudio.model_components.losses import (
distortion_loss,
interlevel_loss,
orientation_loss,
pred_normal_loss,
)
from nerfstudio.model_components.ray_samplers import (
ProposalNetworkSampler,
UniformSampler,
)
from nerfstudio.model_components.renderers import FeatureRenderer
from nerfstudio.model_components.scene_colliders import AABBBoxCollider, SphereCollider
from nerfstudio.models.dreamfusion import DreamFusionModel, DreamFusionModelConfig
from nerfstudio.utils import colormaps, colors, math, misc
@@ -45,7 +38,7 @@ class DreamEmbeddingModelConfig(DreamFusionModelConfig):
_target: Type = field(default_factory=lambda: DreamEmbeddingModel)
"""target class to instantiate"""
prompt: str = "A high quality photo of a frog"
prompt: str = "A high quality photo of a tiger wearing a lab coat"
"""prompt for stable dreamfusion"""
orientation_loss_mult: float = 0.0001
@@ -86,8 +79,18 @@ class DreamEmbeddingModelConfig(DreamFusionModelConfig):
"""Use the same proposal network. Otherwise use different ones."""
proposal_net_args_list: List[Dict] = field(
default_factory=lambda: [
{"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 128},
{"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 256},
{
"hidden_dim": 16,
"log2_hashmap_size": 17,
"num_levels": 5,
"max_res": 128,
},
{
"hidden_dim": 16,
"log2_hashmap_size": 17,
"num_levels": 5,
"max_res": 256,
},
]
)
"""Arguments for the proposal density fields."""
@@ -149,26 +152,43 @@ class DreamEmbeddingModel(DreamFusionModel):
) -> None:
super().__init__(config=config, **kwargs)
def populate_modules(self):
"""Set the fields and modules"""
super().populate_modules()
# setting up fields
self.field = DreamEmbeddingField(self.scene_box.aabb, max_res=self.config.max_res)
self.field = DreamEmbeddingField(
self.scene_box.aabb, max_res=self.config.max_res
)
self.sd = StableDiffusion(self.sd_device, version=self.sd_version)
self.text_embeddings = PositionalTextEmbeddings(
base_prompt=self.cur_prompt,
top_prompt=self.cur_prompt + self.top_prompt,
side_prompt=self.cur_prompt + self.side_prompt,
back_prompt=self.cur_prompt + self.back_prompt,
front_prompt=self.cur_prompt + self.front_prompt,
stable_diffusion=self.sd,
positional_prompting=self.positional_prompting,
)
self.renderer_feature = FeatureRenderer()
def get_outputs(self, ray_bundle: RayBundle, ray_dims: Tuple):
def get_outputs(self, ray_bundle: RayBundle, ray_dims: Tuple):
# uniform sampling
background_feature = self.field.get_background_feature(ray_bundle)
ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns)
ray_samples, weights_list, ray_samples_list = self.proposal_sampler(
ray_bundle, density_fns=self.density_fns
)
field_outputs = self.field(ray_samples, compute_normals=True)
density = field_outputs[FieldHeadNames.DENSITY]
if self.initialize_density:
pos = ray_samples.frustums.get_positions()
density_blob = self.density_strength * torch.exp(-torch.norm(pos, dim=-1) / (2 * 0.04))[..., None]
density_blob = (
self.density_strength
* torch.exp(-torch.norm(pos, dim=-1) / (2 * 0.04))[..., None]
)
density = density + density_blob
weights = ray_samples.get_weights(density)
@@ -177,9 +197,13 @@ class DreamEmbeddingModel(DreamFusionModel):
accumulation = self.renderer_accumulation(weights)
depth = self.renderer_depth(weights, ray_samples)
features = self.renderer_feature(features=field_outputs[FieldHeadNames.FEATURE], weights=weights)
features = self.renderer_feature(
features=field_outputs[FieldHeadNames.FEATURE], weights=weights
)
accum_mask = torch.clamp((torch.nan_to_num(accumulation, nan=0.0)), min=0.0, max=1.0)
accum_mask = torch.clamp(
(torch.nan_to_num(accumulation, nan=0.0)), min=0.0, max=1.0
)
accum_mask_inv = 1.0 - accum_mask
background = accum_mask_inv * background_feature
@@ -198,25 +222,37 @@ class DreamEmbeddingModel(DreamFusionModel):
if ray_dims:
height = ray_dims[0]
width = ray_dims[1]
else:
height = int(latents.shape[0] ** 0.5)
width = int(latents.shape[0] ** 0.5)
# latents_input = latents.view(1, height, width, 4).half()
latents_input = features.view(1, height, width, 4).half()
latents_input = latents_input.permute(0, 3, 1, 2)
# latents_input = torch.nn.functional.interpolate(latents_input, size=(latents_input.shape[2] // 8, latents_input.shape[3] // 8), mode='bilinear')
rgb = self.sd.latents_to_img(latents_input) #.permute(0, 2, 3, 1)[0]
if ray_dims:
rgb = torch.nn.functional.interpolate(rgb, size=(ray_dims[0], ray_dims[1]), mode='bilinear')
# print(height, width)
# ADD WHITE BG TO RGB
latents_input = latents.view(1, height, width, 4).half()
# latents_input = features.view(1, height, width, 4).half()
latents_input = latents_input.permute(0, 3, 1, 2)
# latents_input = torch.nn.functional.interpolate(latents_input, size=(latents_input.shape[2] // 8, latents_input.shape[3] // 8), mode='bilinear')
rgb = self.sd.latents_to_img(latents_input) # .permute(0, 2, 3, 1)[0]
# rgb = torch.nn.functional.interpolate(
# rgb, size=(height, width), mode="bilinear"
# )
rgb = rgb.permute(0, 2, 3, 1)[0]
rgb = torch.flatten(rgb, end_dim=-2)
rgb = rgb.to(torch.float32)
# if ray_dims:
# print(rgb.shape)
outputs["rgb"] = rgb
# ADD WHITE BG TO RGB
white_bg = torch.tensor([1.0, 1.0, 1.0], device=self.device)
a_mask = accum_mask.reshape(1, 1, height, width)
a_mask = torch.nn.functional.interpolate(
a_mask, size=(height * 8, width * 8), mode="bilinear"
)
a_mask = torch.flatten(a_mask.permute(0, 2, 3, 1), end_dim=-2)
a_mask_inv = 1 - a_mask
rgb = rgb.permute(0, 2, 3, 1)[0]
rgb = torch.flatten(rgb, end_dim=-2)
rgb = a_mask * rgb + (a_mask_inv * white_bg)
rgb = rgb.to(torch.float32)
# print("rgb", rgb)
# print(rgb.shape)
outputs["rgb"] = rgb
# These use a lot of GPU memory, so we avoid storing them for eval.
if self.training:
@@ -224,36 +260,51 @@ class DreamEmbeddingModel(DreamFusionModel):
outputs["ray_samples_list"] = ray_samples_list
for i in range(self.config.num_proposal_iterations):
outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i])
outputs[f"prop_depth_{i}"] = self.renderer_depth(
weights=weights_list[i], ray_samples=ray_samples_list[i]
)
normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights)
normals = self.renderer_normals(
normals=field_outputs[FieldHeadNames.NORMALS], weights=weights
)
outputs["normals"] = self.shader_normals(normals, weights=accum_mask)
outputs["rendered_orientation_loss"] = orientation_loss(
weights.detach(), field_outputs[FieldHeadNames.NORMALS], ray_bundle.directions
weights.detach(),
field_outputs[FieldHeadNames.NORMALS],
ray_bundle.directions,
)
assert weights.shape[-1] == 1
if self.config.opacity_penalty:
outputs["opacity_loss"] = torch.sqrt(torch.sum(weights, dim=-2) ** 2 + 0.01) * self.config.opacity_loss_mult
outputs["opacity_loss"] = (
torch.sqrt(torch.sum(weights, dim=-2) ** 2 + 0.01)
* self.config.opacity_loss_mult
)
return outputs
def get_loss_dict(self, outputs, batch, metrics_dict=None) -> Dict[str, torch.Tensor]:
def get_loss_dict(
self, outputs, batch, metrics_dict=None
) -> Dict[str, torch.Tensor]:
# Scaling metrics by coefficients to create the losses.
loss_dict = {}
loss_dict = misc.scale_dict(loss_dict, self.config.loss_coefficients)
if self.train_normals:
# orientation loss for computed normals
loss_dict["orientation_loss"] = self.config.orientation_loss_mult * torch.mean(
loss_dict[
"orientation_loss"
] = self.config.orientation_loss_mult * torch.mean(
outputs["rendered_orientation_loss"]
)
else:
loss_dict["orientation_loss"] = 0
if self.config.opacity_penalty:
loss_dict["opacity_loss"] = self.config.opacity_loss_mult * outputs["opacity_loss"].mean()
loss_dict["opacity_loss"] = (
self.config.opacity_loss_mult * outputs["opacity_loss"].mean()
)
if self.prompt != self.cur_prompt:
self.cur_prompt = self.prompt
@@ -271,7 +322,12 @@ class DreamEmbeddingModel(DreamFusionModel):
latents = (
outputs["latents"]
.view(1, int(outputs["latents"].shape[0] ** 0.5), int(outputs["latents"].shape[0] ** 0.5), 4)
.view(
1,
int(outputs["latents"].shape[0] ** 0.5),
int(outputs["latents"].shape[0] ** 0.5),
4,
)
.permute(0, 3, 1, 2)
)
@@ -285,15 +341,21 @@ class DreamEmbeddingModel(DreamFusionModel):
loss_dict["sds_loss"] = sds_loss.to(self.device)
if self.training:
loss_dict["distortion_loss"] = self.config.distortion_loss_mult * distortion_loss(
loss_dict[
"distortion_loss"
] = self.config.distortion_loss_mult * distortion_loss(
outputs["weights_list"], outputs["ray_samples_list"]
)
loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss(
loss_dict[
"interlevel_loss"
] = self.config.interlevel_loss_mult * interlevel_loss(
outputs["weights_list"], outputs["ray_samples_list"]
)
return loss_dict
def forward(self, ray_bundle: RayBundle, ray_dims: Tuple=()) -> Dict[str, torch.Tensor]:
def forward(
self, ray_bundle: RayBundle, ray_dims: Tuple = ()
) -> Dict[str, torch.Tensor]:
"""Run forward starting with a ray bundle. This outputs different things depending on the configuration
of the model and whether or not the batch is provided (whether or not we are training basically)
@@ -305,9 +367,11 @@ class DreamEmbeddingModel(DreamFusionModel):
ray_bundle = self.collider(ray_bundle)
return self.get_outputs(ray_bundle, ray_dims)
@torch.no_grad()
def get_outputs_for_camera_ray_bundle(self, camera_ray_bundle: RayBundle) -> Dict[str, torch.Tensor]:
def get_outputs_for_camera_ray_bundle(
self, camera_ray_bundle: RayBundle
) -> Dict[str, torch.Tensor]:
"""Takes in camera parameters and computes the output of the model.
Args:
@@ -320,8 +384,12 @@ class DreamEmbeddingModel(DreamFusionModel):
for i in range(0, num_rays, num_rays_per_chunk):
start_idx = i
end_idx = i + num_rays_per_chunk
ray_bundle = camera_ray_bundle.get_row_major_sliced_ray_bundle(start_idx, end_idx)
outputs = self.forward(ray_bundle=ray_bundle, ray_dims=(image_height, image_width))
ray_bundle = camera_ray_bundle.get_row_major_sliced_ray_bundle(
start_idx, end_idx
)
outputs = self.forward(
ray_bundle=ray_bundle, ray_dims=(image_height, image_width)
)
for output_name, output in outputs.items(): # type: ignore
outputs_lists[output_name].append(output)
outputs = {}
@@ -329,11 +397,11 @@ class DreamEmbeddingModel(DreamFusionModel):
if not torch.is_tensor(outputs_list[0]):
# TODO: handle lists of tensors as well
continue
# if output_name == "rgb":
# outputs[output_name] = torch.cat(outputs_list).view(image_height * 8, image_width * 8, -1) # type: ignore
# else:
# outputs[output_name] = torch.cat(outputs_list).view(image_height, image_width, -1) # type: ignore
outputs[output_name] = torch.cat(outputs_list).view(image_height, image_width, -1) # type: ignore
if output_name == "rgb":
outputs[output_name] = torch.cat(outputs_list).view(image_height * 8, image_width * 8, -1) # type: ignore
else:
outputs[output_name] = torch.cat(outputs_list).view(image_height, image_width, -1) # type: ignore
# outputs[output_name] = torch.cat(outputs_list).view(image_height, image_width, -1) # type: ignore
# print(output_name, outputs[output_name].shape)
return outputs
@@ -355,13 +423,20 @@ class DreamEmbeddingModel(DreamFusionModel):
accumulation=outputs["accumulation"],
)
size = outputs["latents"].shape[0]
latents_input = outputs["latents"].view(1, size, size, 4).half()
latents_input = latents_input.permute(0, 3, 1, 2)
rgb = self.sd.latents_to_img(latents_input)
rgb = rgb.permute(0, 2, 3, 1)[0].reshape(512, 512, 3).cpu().to(torch.float32)
metrics_dict = {}
images_dict = {
"img": outputs["rgb"].reshape(512, 512, 3), # this is a bit weird
"img": rgb,
"accumulation": acc,
"depth": depth,
"prop_depth_0": prop_depth_0,
"prop_depth_1": prop_depth_1,
"normals": outputs["normals"],
}
return metrics_dict, images_dict
return metrics_dict, images_dict