mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-18 13:53:44 +08:00
fixed viewer with dream embedding
This commit is contained in:
1 parent
e007d84edd
commit
5a2985c5f5
3 files changed
+188
-78
No files matched your search
@@ -54,7 +54,12 @@ class TrivialDataset(InputDataset):
|
||||
return self.size
|
||||
|
||||
def __getitem__(self, index: int) -> Dict:
|
||||
return {"image": torch.cat([torch.ones(128, 256, 3), torch.zeros(128, 256, 3)], dim=0), "image_idx": index}
|
||||
return {
|
||||
"image": torch.cat(
|
||||
[torch.ones(128, 256, 3), torch.zeros(128, 256, 3)], dim=0
|
||||
),
|
||||
"image_idx": index,
|
||||
}
|
||||
|
||||
|
||||
def random_train_pose(
|
||||
@@ -84,14 +89,19 @@ def random_train_pose(
|
||||
poses: [size, 4, 4]
|
||||
"""
|
||||
|
||||
vertical_rotation_range = (vertical_rotation_range[0] + 90, vertical_rotation_range[1] + 90)
|
||||
vertical_rotation_range = (
|
||||
vertical_rotation_range[0] + 90,
|
||||
vertical_rotation_range[1] + 90,
|
||||
)
|
||||
# This is the uniform sample on the part of the sphere we care about where 0 = 0 degrees and 1 = 360 degrees
|
||||
sampled_uniform = (
|
||||
torch.rand(size) * (vertical_rotation_range[1] - vertical_rotation_range[0]) + vertical_rotation_range[0]
|
||||
torch.rand(size) * (vertical_rotation_range[1] - vertical_rotation_range[0])
|
||||
+ vertical_rotation_range[0]
|
||||
) / 180
|
||||
vertical_rotation = torch.arccos(1 - 2 * sampled_uniform)
|
||||
central_rotation = torch.deg2rad(
|
||||
torch.rand(size) * (central_rotation_range[1] - central_rotation_range[0]) + central_rotation_range[0]
|
||||
torch.rand(size) * (central_rotation_range[1] - central_rotation_range[0])
|
||||
+ central_rotation_range[0]
|
||||
)
|
||||
|
||||
c_cos = torch.cos(central_rotation)
|
||||
@@ -121,7 +131,9 @@ def random_train_pose(
|
||||
|
||||
# Default directions are facing in the -z direction, so origins should face opposite way
|
||||
origins = torch.stack([torch.tensor([0, 0, 1])] * size, dim=0)
|
||||
origins = (origins * radius_mean) + (origins * (torch.randn((origins.shape)) * radius_std))
|
||||
origins = (origins * radius_mean) + (
|
||||
origins * (torch.randn((origins.shape)) * radius_std)
|
||||
)
|
||||
R = torch.bmm(rot_z, rot_x) # Want to have Rx @ Ry @ origin
|
||||
t = (
|
||||
torch.bmm(R, origins.unsqueeze(-1))
|
||||
@@ -158,7 +170,7 @@ class DreamFusionDataManagerConfig(DataManagerConfig):
|
||||
"""Number of images per batch for training"""
|
||||
eval_images_per_batch: int = 1
|
||||
"""Number of images per batch for evaluation"""
|
||||
radius_mean: float = 3.0
|
||||
radius_mean: float = 2.5
|
||||
"""Mean radius of camera orbit"""
|
||||
radius_std: float = 0.1
|
||||
"""Std of radius of camera orbit"""
|
||||
@@ -208,7 +220,9 @@ class DreamFusionDataManager(DataManager): # pylint: disable=abstract-method
|
||||
self.test_split = "test" if test_mode in ["test", "inference"] else "val"
|
||||
|
||||
if self.config.data is not None:
|
||||
CONSOLE.print("[red] --data should not be used with the DreamFusionDataManager[/red]")
|
||||
CONSOLE.print(
|
||||
"[red] --data should not be used with the DreamFusionDataManager[/red]"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
cameras, _, _ = random_train_pose(
|
||||
@@ -265,7 +279,9 @@ class DreamFusionDataManager(DataManager): # pylint: disable=abstract-method
|
||||
center=self.config.center,
|
||||
central_rotation_range=(-horizontal_range, horizontal_range),
|
||||
)
|
||||
ray_bundle = cameras.generate_rays(torch.tensor(list(range(self.config.train_images_per_batch)))).flatten()
|
||||
ray_bundle = cameras.generate_rays(
|
||||
torch.tensor(list(range(self.config.train_images_per_batch)))
|
||||
).flatten()
|
||||
|
||||
# camera_idx = torch.randint(0, self.eval_cameras.shape[0], [1], dtype=torch.long, device=self.device)
|
||||
# ray_bundle = self.eval_cameras.generate_rays(camera_idx).flatten()
|
||||
@@ -310,7 +326,9 @@ class DreamFusionDataManager(DataManager): # pylint: disable=abstract-method
|
||||
def get_eval_rays_per_batch(self) -> int:
|
||||
return self.config.eval_resolution**2
|
||||
|
||||
def get_param_groups(self) -> Dict[str, List[Parameter]]: # pylint: disable=no-self-use
|
||||
def get_param_groups(
|
||||
self,
|
||||
) -> Dict[str, List[Parameter]]: # pylint: disable=no-self-use
|
||||
"""Get the param groups for the data manager.
|
||||
Returns:
|
||||
A list of dictionaries containing the data manager's param groups.
|
||||
|
||||
@@ -44,9 +44,13 @@ class ModelConfig(InstantiateConfig):
|
||||
"""target class to instantiate"""
|
||||
enable_collider: bool = True
|
||||
"""Whether to create a scene collider to filter rays."""
|
||||
collider_params: Optional[Dict[str, float]] = to_immutable_dict({"near_plane": 2.0, "far_plane": 6.0})
|
||||
collider_params: Optional[Dict[str, float]] = to_immutable_dict(
|
||||
{"near_plane": 2.0, "far_plane": 6.0}
|
||||
)
|
||||
"""parameters to instantiate scene collider with"""
|
||||
loss_coefficients: Dict[str, float] = to_immutable_dict({"rgb_loss_coarse": 1.0, "rgb_loss_fine": 1.0})
|
||||
loss_coefficients: Dict[str, float] = to_immutable_dict(
|
||||
{"rgb_loss_coarse": 1.0, "rgb_loss_fine": 1.0}
|
||||
)
|
||||
"""parameters to instantiate density field with"""
|
||||
eval_num_rays_per_chunk: int = 4096
|
||||
"""specifies number of rays per chunk during eval"""
|
||||
@@ -76,7 +80,9 @@ class Model(nn.Module):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.scene_box = scene_box
|
||||
self.render_aabb = None # the box that we want to render - should be a subset of scene_box
|
||||
self.render_aabb = (
|
||||
None # the box that we want to render - should be a subset of scene_box
|
||||
)
|
||||
self.num_train_data = num_train_data
|
||||
self.kwargs = kwargs
|
||||
self.collider = None
|
||||
@@ -92,7 +98,8 @@ class Model(nn.Module):
|
||||
return self.device_indicator_param.device
|
||||
|
||||
def get_training_callbacks( # pylint:disable=no-self-use
|
||||
self, training_callback_attributes: TrainingCallbackAttributes # pylint: disable=unused-argument
|
||||
self,
|
||||
training_callback_attributes: TrainingCallbackAttributes, # pylint: disable=unused-argument
|
||||
) -> List[TrainingCallback]:
|
||||
"""Returns a list of callbacks that run functions at the specified training iterations."""
|
||||
return []
|
||||
@@ -104,7 +111,8 @@ class Model(nn.Module):
|
||||
|
||||
if self.config.enable_collider:
|
||||
self.collider = NearFarCollider(
|
||||
near_plane=self.config.collider_params["near_plane"], far_plane=self.config.collider_params["far_plane"]
|
||||
near_plane=self.config.collider_params["near_plane"],
|
||||
far_plane=self.config.collider_params["far_plane"],
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
@@ -152,7 +160,9 @@ class Model(nn.Module):
|
||||
return {}
|
||||
|
||||
@abstractmethod
|
||||
def get_loss_dict(self, outputs, batch, metrics_dict=None) -> Dict[str, torch.Tensor]:
|
||||
def get_loss_dict(
|
||||
self, outputs, batch, metrics_dict=None
|
||||
) -> Dict[str, torch.Tensor]:
|
||||
"""Computes and returns the losses dict.
|
||||
|
||||
Args:
|
||||
@@ -162,7 +172,9 @@ class Model(nn.Module):
|
||||
"""
|
||||
|
||||
@torch.no_grad()
|
||||
def get_outputs_for_camera_ray_bundle(self, camera_ray_bundle: RayBundle) -> Dict[str, torch.Tensor]:
|
||||
def get_outputs_for_camera_ray_bundle(
|
||||
self, camera_ray_bundle: RayBundle
|
||||
) -> Dict[str, torch.Tensor]:
|
||||
"""Takes in camera parameters and computes the output of the model.
|
||||
|
||||
Args:
|
||||
@@ -175,7 +187,9 @@ class Model(nn.Module):
|
||||
for i in range(0, num_rays, num_rays_per_chunk):
|
||||
start_idx = i
|
||||
end_idx = i + num_rays_per_chunk
|
||||
ray_bundle = camera_ray_bundle.get_row_major_sliced_ray_bundle(start_idx, end_idx)
|
||||
ray_bundle = camera_ray_bundle.get_row_major_sliced_ray_bundle(
|
||||
start_idx, end_idx
|
||||
)
|
||||
outputs = self.forward(ray_bundle=ray_bundle)
|
||||
for output_name, output in outputs.items(): # type: ignore
|
||||
outputs_lists[output_name].append(output)
|
||||
@@ -210,7 +224,10 @@ class Model(nn.Module):
|
||||
Args:
|
||||
loaded_state: dictionary of pre-trained model states
|
||||
"""
|
||||
state = {key.replace("module.", ""): value for key, value in loaded_state["model"].items()}
|
||||
state = {
|
||||
key.replace("module.", ""): value
|
||||
for key, value in loaded_state["model"].items()
|
||||
}
|
||||
self.load_state_dict(state) # type: ignore
|
||||
|
||||
def update_to_step(self, step: int) -> None:
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Optional, Tuple, Type
|
||||
|
||||
import mediapy
|
||||
import numpy as np
|
||||
|
||||
import torch
|
||||
from torch.nn import Parameter
|
||||
|
||||
from typing_extensions import Literal
|
||||
|
||||
from nerfstudio.cameras.rays import RayBundle
|
||||
@@ -27,14 +25,9 @@ from nerfstudio.model_components.losses import (
|
||||
distortion_loss,
|
||||
interlevel_loss,
|
||||
orientation_loss,
|
||||
pred_normal_loss,
|
||||
)
|
||||
from nerfstudio.model_components.ray_samplers import (
|
||||
ProposalNetworkSampler,
|
||||
UniformSampler,
|
||||
)
|
||||
|
||||
from nerfstudio.model_components.renderers import FeatureRenderer
|
||||
from nerfstudio.model_components.scene_colliders import AABBBoxCollider, SphereCollider
|
||||
from nerfstudio.models.dreamfusion import DreamFusionModel, DreamFusionModelConfig
|
||||
from nerfstudio.utils import colormaps, colors, math, misc
|
||||
|
||||
@@ -45,7 +38,7 @@ class DreamEmbeddingModelConfig(DreamFusionModelConfig):
|
||||
|
||||
_target: Type = field(default_factory=lambda: DreamEmbeddingModel)
|
||||
"""target class to instantiate"""
|
||||
prompt: str = "A high quality photo of a frog"
|
||||
prompt: str = "A high quality photo of a tiger wearing a lab coat"
|
||||
"""prompt for stable dreamfusion"""
|
||||
|
||||
orientation_loss_mult: float = 0.0001
|
||||
@@ -86,8 +79,18 @@ class DreamEmbeddingModelConfig(DreamFusionModelConfig):
|
||||
"""Use the same proposal network. Otherwise use different ones."""
|
||||
proposal_net_args_list: List[Dict] = field(
|
||||
default_factory=lambda: [
|
||||
{"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 128},
|
||||
{"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 256},
|
||||
{
|
||||
"hidden_dim": 16,
|
||||
"log2_hashmap_size": 17,
|
||||
"num_levels": 5,
|
||||
"max_res": 128,
|
||||
},
|
||||
{
|
||||
"hidden_dim": 16,
|
||||
"log2_hashmap_size": 17,
|
||||
"num_levels": 5,
|
||||
"max_res": 256,
|
||||
},
|
||||
]
|
||||
)
|
||||
"""Arguments for the proposal density fields."""
|
||||
@@ -149,26 +152,43 @@ class DreamEmbeddingModel(DreamFusionModel):
|
||||
) -> None:
|
||||
super().__init__(config=config, **kwargs)
|
||||
|
||||
|
||||
def populate_modules(self):
|
||||
"""Set the fields and modules"""
|
||||
super().populate_modules()
|
||||
|
||||
# setting up fields
|
||||
self.field = DreamEmbeddingField(self.scene_box.aabb, max_res=self.config.max_res)
|
||||
self.field = DreamEmbeddingField(
|
||||
self.scene_box.aabb, max_res=self.config.max_res
|
||||
)
|
||||
|
||||
self.sd = StableDiffusion(self.sd_device, version=self.sd_version)
|
||||
self.text_embeddings = PositionalTextEmbeddings(
|
||||
base_prompt=self.cur_prompt,
|
||||
top_prompt=self.cur_prompt + self.top_prompt,
|
||||
side_prompt=self.cur_prompt + self.side_prompt,
|
||||
back_prompt=self.cur_prompt + self.back_prompt,
|
||||
front_prompt=self.cur_prompt + self.front_prompt,
|
||||
stable_diffusion=self.sd,
|
||||
positional_prompting=self.positional_prompting,
|
||||
)
|
||||
|
||||
self.renderer_feature = FeatureRenderer()
|
||||
|
||||
def get_outputs(self, ray_bundle: RayBundle, ray_dims: Tuple):
|
||||
def get_outputs(self, ray_bundle: RayBundle, ray_dims: Tuple):
|
||||
# uniform sampling
|
||||
background_feature = self.field.get_background_feature(ray_bundle)
|
||||
ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns)
|
||||
ray_samples, weights_list, ray_samples_list = self.proposal_sampler(
|
||||
ray_bundle, density_fns=self.density_fns
|
||||
)
|
||||
field_outputs = self.field(ray_samples, compute_normals=True)
|
||||
density = field_outputs[FieldHeadNames.DENSITY]
|
||||
|
||||
if self.initialize_density:
|
||||
pos = ray_samples.frustums.get_positions()
|
||||
density_blob = self.density_strength * torch.exp(-torch.norm(pos, dim=-1) / (2 * 0.04))[..., None]
|
||||
density_blob = (
|
||||
self.density_strength
|
||||
* torch.exp(-torch.norm(pos, dim=-1) / (2 * 0.04))[..., None]
|
||||
)
|
||||
density = density + density_blob
|
||||
|
||||
weights = ray_samples.get_weights(density)
|
||||
@@ -177,9 +197,13 @@ class DreamEmbeddingModel(DreamFusionModel):
|
||||
|
||||
accumulation = self.renderer_accumulation(weights)
|
||||
depth = self.renderer_depth(weights, ray_samples)
|
||||
features = self.renderer_feature(features=field_outputs[FieldHeadNames.FEATURE], weights=weights)
|
||||
features = self.renderer_feature(
|
||||
features=field_outputs[FieldHeadNames.FEATURE], weights=weights
|
||||
)
|
||||
|
||||
accum_mask = torch.clamp((torch.nan_to_num(accumulation, nan=0.0)), min=0.0, max=1.0)
|
||||
accum_mask = torch.clamp(
|
||||
(torch.nan_to_num(accumulation, nan=0.0)), min=0.0, max=1.0
|
||||
)
|
||||
accum_mask_inv = 1.0 - accum_mask
|
||||
|
||||
background = accum_mask_inv * background_feature
|
||||
@@ -198,25 +222,37 @@ class DreamEmbeddingModel(DreamFusionModel):
|
||||
if ray_dims:
|
||||
height = ray_dims[0]
|
||||
width = ray_dims[1]
|
||||
else:
|
||||
height = int(latents.shape[0] ** 0.5)
|
||||
width = int(latents.shape[0] ** 0.5)
|
||||
# latents_input = latents.view(1, height, width, 4).half()
|
||||
latents_input = features.view(1, height, width, 4).half()
|
||||
latents_input = latents_input.permute(0, 3, 1, 2)
|
||||
# latents_input = torch.nn.functional.interpolate(latents_input, size=(latents_input.shape[2] // 8, latents_input.shape[3] // 8), mode='bilinear')
|
||||
rgb = self.sd.latents_to_img(latents_input) #.permute(0, 2, 3, 1)[0]
|
||||
if ray_dims:
|
||||
rgb = torch.nn.functional.interpolate(rgb, size=(ray_dims[0], ray_dims[1]), mode='bilinear')
|
||||
# print(height, width)
|
||||
|
||||
# ADD WHITE BG TO RGB
|
||||
latents_input = latents.view(1, height, width, 4).half()
|
||||
# latents_input = features.view(1, height, width, 4).half()
|
||||
latents_input = latents_input.permute(0, 3, 1, 2)
|
||||
# latents_input = torch.nn.functional.interpolate(latents_input, size=(latents_input.shape[2] // 8, latents_input.shape[3] // 8), mode='bilinear')
|
||||
rgb = self.sd.latents_to_img(latents_input) # .permute(0, 2, 3, 1)[0]
|
||||
# rgb = torch.nn.functional.interpolate(
|
||||
# rgb, size=(height, width), mode="bilinear"
|
||||
# )
|
||||
|
||||
rgb = rgb.permute(0, 2, 3, 1)[0]
|
||||
rgb = torch.flatten(rgb, end_dim=-2)
|
||||
rgb = rgb.to(torch.float32)
|
||||
# if ray_dims:
|
||||
# print(rgb.shape)
|
||||
outputs["rgb"] = rgb
|
||||
# ADD WHITE BG TO RGB
|
||||
white_bg = torch.tensor([1.0, 1.0, 1.0], device=self.device)
|
||||
|
||||
a_mask = accum_mask.reshape(1, 1, height, width)
|
||||
|
||||
a_mask = torch.nn.functional.interpolate(
|
||||
a_mask, size=(height * 8, width * 8), mode="bilinear"
|
||||
)
|
||||
|
||||
a_mask = torch.flatten(a_mask.permute(0, 2, 3, 1), end_dim=-2)
|
||||
a_mask_inv = 1 - a_mask
|
||||
|
||||
rgb = rgb.permute(0, 2, 3, 1)[0]
|
||||
rgb = torch.flatten(rgb, end_dim=-2)
|
||||
rgb = a_mask * rgb + (a_mask_inv * white_bg)
|
||||
rgb = rgb.to(torch.float32)
|
||||
# print("rgb", rgb)
|
||||
# print(rgb.shape)
|
||||
|
||||
outputs["rgb"] = rgb
|
||||
|
||||
# These use a lot of GPU memory, so we avoid storing them for eval.
|
||||
if self.training:
|
||||
@@ -224,36 +260,51 @@ class DreamEmbeddingModel(DreamFusionModel):
|
||||
outputs["ray_samples_list"] = ray_samples_list
|
||||
|
||||
for i in range(self.config.num_proposal_iterations):
|
||||
outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i])
|
||||
outputs[f"prop_depth_{i}"] = self.renderer_depth(
|
||||
weights=weights_list[i], ray_samples=ray_samples_list[i]
|
||||
)
|
||||
|
||||
normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights)
|
||||
normals = self.renderer_normals(
|
||||
normals=field_outputs[FieldHeadNames.NORMALS], weights=weights
|
||||
)
|
||||
outputs["normals"] = self.shader_normals(normals, weights=accum_mask)
|
||||
|
||||
outputs["rendered_orientation_loss"] = orientation_loss(
|
||||
weights.detach(), field_outputs[FieldHeadNames.NORMALS], ray_bundle.directions
|
||||
weights.detach(),
|
||||
field_outputs[FieldHeadNames.NORMALS],
|
||||
ray_bundle.directions,
|
||||
)
|
||||
|
||||
assert weights.shape[-1] == 1
|
||||
if self.config.opacity_penalty:
|
||||
outputs["opacity_loss"] = torch.sqrt(torch.sum(weights, dim=-2) ** 2 + 0.01) * self.config.opacity_loss_mult
|
||||
outputs["opacity_loss"] = (
|
||||
torch.sqrt(torch.sum(weights, dim=-2) ** 2 + 0.01)
|
||||
* self.config.opacity_loss_mult
|
||||
)
|
||||
|
||||
return outputs
|
||||
|
||||
def get_loss_dict(self, outputs, batch, metrics_dict=None) -> Dict[str, torch.Tensor]:
|
||||
def get_loss_dict(
|
||||
self, outputs, batch, metrics_dict=None
|
||||
) -> Dict[str, torch.Tensor]:
|
||||
# Scaling metrics by coefficients to create the losses.
|
||||
|
||||
loss_dict = {}
|
||||
loss_dict = misc.scale_dict(loss_dict, self.config.loss_coefficients)
|
||||
if self.train_normals:
|
||||
# orientation loss for computed normals
|
||||
loss_dict["orientation_loss"] = self.config.orientation_loss_mult * torch.mean(
|
||||
loss_dict[
|
||||
"orientation_loss"
|
||||
] = self.config.orientation_loss_mult * torch.mean(
|
||||
outputs["rendered_orientation_loss"]
|
||||
)
|
||||
else:
|
||||
loss_dict["orientation_loss"] = 0
|
||||
|
||||
if self.config.opacity_penalty:
|
||||
loss_dict["opacity_loss"] = self.config.opacity_loss_mult * outputs["opacity_loss"].mean()
|
||||
loss_dict["opacity_loss"] = (
|
||||
self.config.opacity_loss_mult * outputs["opacity_loss"].mean()
|
||||
)
|
||||
|
||||
if self.prompt != self.cur_prompt:
|
||||
self.cur_prompt = self.prompt
|
||||
@@ -271,7 +322,12 @@ class DreamEmbeddingModel(DreamFusionModel):
|
||||
|
||||
latents = (
|
||||
outputs["latents"]
|
||||
.view(1, int(outputs["latents"].shape[0] ** 0.5), int(outputs["latents"].shape[0] ** 0.5), 4)
|
||||
.view(
|
||||
1,
|
||||
int(outputs["latents"].shape[0] ** 0.5),
|
||||
int(outputs["latents"].shape[0] ** 0.5),
|
||||
4,
|
||||
)
|
||||
.permute(0, 3, 1, 2)
|
||||
)
|
||||
|
||||
@@ -285,15 +341,21 @@ class DreamEmbeddingModel(DreamFusionModel):
|
||||
loss_dict["sds_loss"] = sds_loss.to(self.device)
|
||||
|
||||
if self.training:
|
||||
loss_dict["distortion_loss"] = self.config.distortion_loss_mult * distortion_loss(
|
||||
loss_dict[
|
||||
"distortion_loss"
|
||||
] = self.config.distortion_loss_mult * distortion_loss(
|
||||
outputs["weights_list"], outputs["ray_samples_list"]
|
||||
)
|
||||
loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss(
|
||||
loss_dict[
|
||||
"interlevel_loss"
|
||||
] = self.config.interlevel_loss_mult * interlevel_loss(
|
||||
outputs["weights_list"], outputs["ray_samples_list"]
|
||||
)
|
||||
return loss_dict
|
||||
|
||||
def forward(self, ray_bundle: RayBundle, ray_dims: Tuple=()) -> Dict[str, torch.Tensor]:
|
||||
|
||||
def forward(
|
||||
self, ray_bundle: RayBundle, ray_dims: Tuple = ()
|
||||
) -> Dict[str, torch.Tensor]:
|
||||
"""Run forward starting with a ray bundle. This outputs different things depending on the configuration
|
||||
of the model and whether or not the batch is provided (whether or not we are training basically)
|
||||
|
||||
@@ -305,9 +367,11 @@ class DreamEmbeddingModel(DreamFusionModel):
|
||||
ray_bundle = self.collider(ray_bundle)
|
||||
|
||||
return self.get_outputs(ray_bundle, ray_dims)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def get_outputs_for_camera_ray_bundle(self, camera_ray_bundle: RayBundle) -> Dict[str, torch.Tensor]:
|
||||
def get_outputs_for_camera_ray_bundle(
|
||||
self, camera_ray_bundle: RayBundle
|
||||
) -> Dict[str, torch.Tensor]:
|
||||
"""Takes in camera parameters and computes the output of the model.
|
||||
|
||||
Args:
|
||||
@@ -320,8 +384,12 @@ class DreamEmbeddingModel(DreamFusionModel):
|
||||
for i in range(0, num_rays, num_rays_per_chunk):
|
||||
start_idx = i
|
||||
end_idx = i + num_rays_per_chunk
|
||||
ray_bundle = camera_ray_bundle.get_row_major_sliced_ray_bundle(start_idx, end_idx)
|
||||
outputs = self.forward(ray_bundle=ray_bundle, ray_dims=(image_height, image_width))
|
||||
ray_bundle = camera_ray_bundle.get_row_major_sliced_ray_bundle(
|
||||
start_idx, end_idx
|
||||
)
|
||||
outputs = self.forward(
|
||||
ray_bundle=ray_bundle, ray_dims=(image_height, image_width)
|
||||
)
|
||||
for output_name, output in outputs.items(): # type: ignore
|
||||
outputs_lists[output_name].append(output)
|
||||
outputs = {}
|
||||
@@ -329,11 +397,11 @@ class DreamEmbeddingModel(DreamFusionModel):
|
||||
if not torch.is_tensor(outputs_list[0]):
|
||||
# TODO: handle lists of tensors as well
|
||||
continue
|
||||
# if output_name == "rgb":
|
||||
# outputs[output_name] = torch.cat(outputs_list).view(image_height * 8, image_width * 8, -1) # type: ignore
|
||||
# else:
|
||||
# outputs[output_name] = torch.cat(outputs_list).view(image_height, image_width, -1) # type: ignore
|
||||
outputs[output_name] = torch.cat(outputs_list).view(image_height, image_width, -1) # type: ignore
|
||||
if output_name == "rgb":
|
||||
outputs[output_name] = torch.cat(outputs_list).view(image_height * 8, image_width * 8, -1) # type: ignore
|
||||
else:
|
||||
outputs[output_name] = torch.cat(outputs_list).view(image_height, image_width, -1) # type: ignore
|
||||
# outputs[output_name] = torch.cat(outputs_list).view(image_height, image_width, -1) # type: ignore
|
||||
# print(output_name, outputs[output_name].shape)
|
||||
|
||||
return outputs
|
||||
@@ -355,13 +423,20 @@ class DreamEmbeddingModel(DreamFusionModel):
|
||||
accumulation=outputs["accumulation"],
|
||||
)
|
||||
|
||||
size = outputs["latents"].shape[0]
|
||||
|
||||
latents_input = outputs["latents"].view(1, size, size, 4).half()
|
||||
latents_input = latents_input.permute(0, 3, 1, 2)
|
||||
rgb = self.sd.latents_to_img(latents_input)
|
||||
rgb = rgb.permute(0, 2, 3, 1)[0].reshape(512, 512, 3).cpu().to(torch.float32)
|
||||
|
||||
metrics_dict = {}
|
||||
images_dict = {
|
||||
"img": outputs["rgb"].reshape(512, 512, 3), # this is a bit weird
|
||||
"img": rgb,
|
||||
"accumulation": acc,
|
||||
"depth": depth,
|
||||
"prop_depth_0": prop_depth_0,
|
||||
"prop_depth_1": prop_depth_1,
|
||||
"normals": outputs["normals"],
|
||||
}
|
||||
return metrics_dict, images_dict
|
||||
return metrics_dict, images_dict
|
||||
Reference in new issue
Block a user