This commit is contained in:
terrance committed 2023-02-21 20:23:05 +00:00
1 parent f7789882b9
commit 0a4d49c4a4
4 files changed
+614 -44

No files matched your search

+49
View File
@@ -30,6 +30,7 @@ from nerfstudio.data.datamanagers.depth_datamanager import DepthDataManagerConfi
from nerfstudio.data.datamanagers.dreamfusion_datamanager import (
DreamFusionDataManagerConfig,
)
from nerfstudio.data.datamanagers.dreamfusionplus_datamanager import DreamFusionPlusDataManagerConfig
from nerfstudio.data.datamanagers.semantic_datamanager import SemanticDataManagerConfig
from nerfstudio.data.datamanagers.variable_res_datamanager import (
VariableResDataManagerConfig,
@@ -51,6 +52,7 @@ from nerfstudio.engine.trainer import TrainerConfig
from nerfstudio.field_components.temporal_distortions import TemporalDistortionKind
from nerfstudio.models.depth_nerfacto import DepthNerfactoModelConfig
from nerfstudio.models.dreamfusion import DreamFusionModelConfig
from nerfstudio.models.dreamfusionplus import DreamFusionPlusModelConfig
from nerfstudio.models.instant_ngp import InstantNGPModelConfig
from nerfstudio.models.mipnerf import MipNerfModel
from nerfstudio.models.nerfacto import NerfactoModelConfig
@@ -383,6 +385,53 @@ method_configs["dreamfusion"] = TrainerConfig(
vis="viewer",
)
method_configs["dreamplus"] = TrainerConfig(
method_name="dreamplus",
steps_per_eval_batch=50,
steps_per_eval_image=50,
steps_per_save=200,
max_num_iterations=30000,
mixed_precision=True,
pipeline=VanillaPipelineConfig(
generative=True,
datamanager=DreamFusionPlusDataManagerConfig(
dataparser=NerfstudioDataParserConfig(),
train_num_rays_per_batch=4096,
eval_num_rays_per_batch=4096,
),
model=DreamFusionPlusModelConfig(
eval_num_rays_per_chunk=1 << 15,
distortion_loss_mult=0.02,
interlevel_loss_mult=1.0,
orientation_loss_mult=0.1,
max_res=256,
sphere_collider=True,
initialize_density=False,
random_background=True,
proposal_warmup=1000,
proposal_update_every=2,
proposal_weights_anneal_max_num_iters=100,
start_lambertian_training=1000,
start_normals_training=500,
opacity_loss_mult=0.001,
prompting_type="location_based",
guidance_scale=100,
),
),
optimizers={
"proposal_networks": {
"optimizer": AdamOptimizerConfig(lr=5e-2, eps=1e-15),
"scheduler": None,
},
"fields": {
"optimizer": AdamOptimizerConfig(lr=5e-3, eps=1e-15),
"scheduler": None,
},
},
viewer=ViewerConfig(num_rays_per_chunk=1 << 15),
vis="viewer",
)
method_configs["nerfplayer-nerfacto"] = TrainerConfig(
method_name="nerfplayer-nerfacto",
pipeline=VanillaPipelineConfig(
@@ -0,0 +1,338 @@
# Copyright 2022 The Nerfstudio Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Data manager for dreamfusion
"""
from __future__ import annotations
import copy
from dataclasses import dataclass, field
from pathlib import Path
from typing import Dict, List, Optional, Tuple, Type, Union
import numpy as np
import torch
from PIL import Image
from rich.progress import Console
from torch.nn import Parameter
from typing_extensions import Literal
from nerfstudio.cameras.cameras import Cameras
from nerfstudio.cameras.rays import RayBundle
from nerfstudio.configs.config_utils import to_immutable_dict
from nerfstudio.data.datamanagers.base_datamanager import DataManager
from nerfstudio.data.datamanagers.dreamfusion_datamanager import (
DreamFusionDataManager,
DreamFusionDataManagerConfig,
random_train_pose,
)
from nerfstudio.data.datasets.base_dataset import InputDataset
from nerfstudio.data.pixel_samplers import PixelSampler
from nerfstudio.data.scene_box import SceneBox
CONSOLE = Console(width=120)
def centralized_jitter(c2w, jitter_std):
assert c2w.shape[0] == 1
theta = torch.randn(1) * jitter_std
phi = torch.randn(1) * jitter_std
R = torch.Tensor(
[
[torch.cos(theta), torch.sin(theta), 0],
[-torch.sin(theta), torch.cos(theta), 0],
[0, 0, 1],
]
)
R = R @ torch.Tensor(
[
[1, 0, 0],
[0, torch.cos(phi), -torch.sin(phi)],
[0, torch.sin(phi), torch.cos(phi)],
]
)
R = R.to(c2w.device)
jittered = copy.deepcopy(c2w)
jittered[..., :3, :3] = R @ jittered[..., :3, :3]
jittered[..., :3, 3] = (R @ jittered[..., :3, 3].unsqueeze(-1)).squeeze(-1)
return jittered
class TrivialDataset(InputDataset):
"""A trivial dataset with blank images for the viewer"""
# pylint: disable=super-init-not-called
def __init__(self, cameras: Cameras, input_img: torch.Tensor):
self.size = cameras.size
self.cameras = cameras
self.alpha_color = None
self.scene_box = SceneBox(torch.Tensor([[-1, -1, -1], [1, 1, 1]]))
self.mask_filenames = None
self.metadata = to_immutable_dict({})
self.input_img = input_img
def __len__(self):
return self.size
def __getitem__(self, index: int) -> Dict:
if index == 0:
return {"image": self.input_img, "image_idx": index}
return {"image": torch.cat([torch.ones(128, 256, 3), torch.zeros(128, 256, 3)], dim=0), "image_idx": index}
@dataclass
class DreamFusionPlusDataManagerConfig(DreamFusionDataManagerConfig):
"""Configuration for data manager instantiation; DataManager is in charge of keeping the train/eval dataparsers;
After instantiation, data manager holds both train/eval datasets and is in charge of returning unpacked
train/eval data at each iteration
"""
_target: Type = field(default_factory=lambda: DreamFusionPlusDataManager)
input_image_path: Path = Path("data/generative/chair/chair.jpg")
"""Directory specifying location of input image."""
input_image_mask_path: Optional[Path] = None
"""Directory specifying location of input image mask, if one exists"""
radius_mean: float = 2.2
"""Mean radius of the camera sphere"""
use_input_transient: bool = False
"""Whether to allow the input image to mask out parts of the image that give poor rgb loss"""
input_distance: float = 1.5
"""Scalar multiple of radius_mean dictating how close we think this image was taken to the object in question.
Large values mean this image was taken relatively far away, small values mean this was taken from relatively
close to the object"""
input_image_position_jitter_std: float = 0
"""Whether or not to jitter the position of the input image each time to prevent accumulation directly
in front of the camera. If this is nonozero, it is the standard deviation of the jitter"""
reference_directory: Path = Path("")
"""Directory specifying location of reference images."""
anneal_camera_angles: float = 0
"""number of steps to anneal camera angles. We start near the input image and slowly work our way outwards
as we train with this greater than 0"""
central_rotation_range: Tuple[float, float] = (-180, 180)
"""Range of central rotation"""
class DreamFusionPlusDataManager(DreamFusionDataManager): # pylint: disable=abstract-method
"""Basic stored data manager implementation.
This is pretty much a port over from our old dataloading utilities, and is a little jank
under the hood. We may clean this up a little bit under the hood with more standard dataloading
components that can be strung together, but it can be just used as a black box for now since
only the constructor is likely to change in the future, or maybe passing in step number to the
next_train and next_eval functions.
Args:
config: the DataManagerConfig used to instantiate class
"""
config: DreamFusionPlusDataManagerConfig
# pylint: disable=super-init-not-called
def __init__(
self,
config: DreamFusionPlusDataManagerConfig,
device: Union[torch.device, str] = "cpu",
test_mode: Literal["test", "val", "inference"] = "val",
world_size: int = 1,
local_rank: int = 0,
**kwargs, # pylint: disable=unused-argument
):
self.config = config
self.device = device
self.world_size = world_size
self.local_rank = local_rank
self.sampler = None
self.test_mode = test_mode
self.test_split = "test" if test_mode in ["test", "inference"] else "val"
self.dataparser = self.config.dataparser.setup()
pil_image = Image.open(self.config.input_image_path)
image = np.array(pil_image, dtype="uint8") # shape is (h, w) or (h, w, 3 or 4)
if len(image.shape) == 2:
image = image[:, :, None].repeat(3, axis=2)
assert image.shape[-1] == 3, "Image must be RGB"
image = torch.from_numpy(image).to(self.device).float() / 255.0
self.input_image = image.to(device)
self.input_image_mask = None
if self.config.input_image_mask_path is not None:
pil_mask = Image.open(self.config.input_image_mask_path)
mask = torch.from_numpy(np.array(pil_mask))
self.input_image_mask = torch.any(mask != 0, dim=-1)
# Transients for the input image, allows it to mask out parts of the image that are background
original_cameras, _, _ = random_train_pose(
1,
1,
device=torch.device("cpu"),
radius_mean=self.config.radius_mean * self.config.input_distance,
radius_std=0,
focal_range=[1, 1],
vertical_rotation_range=[0, 0],
central_rotation_range=[0, 0],
jitter_std=0,
)
original_cameras = Cameras(
camera_to_worlds=original_cameras.camera_to_worlds,
fx=float(image.shape[1]),
fy=float(image.shape[0]),
cx=image.shape[1] / 2,
cy=image.shape[0] / 2,
)
self.original_cameras = original_cameras.to(self.device)
cameras, _, _ = random_train_pose(
self.config.num_eval_angles,
self.config.eval_resolution,
device=self.device,
radius_mean=self.config.radius_mean,
radius_std=self.config.radius_std,
focal_range=self.config.focal_range,
vertical_rotation_range=self.config.vertical_rotation_range,
central_rotation_range=self.config.central_rotation_range,
jitter_std=self.config.jitter_std,
)
cameras = cameras.to("cpu")
cameras = Cameras(
camera_to_worlds=torch.cat([original_cameras.camera_to_worlds, cameras.camera_to_worlds], dim=0),
fx=torch.cat([original_cameras.fx, cameras.fx], dim=0),
fy=torch.cat([original_cameras.fy, cameras.fy], dim=0),
cx=torch.cat([original_cameras.cx, cameras.cx], dim=0),
cy=torch.cat([original_cameras.cy, cameras.cy], dim=0),
width=torch.cat([original_cameras.width, cameras.width], dim=0),
height=torch.cat([original_cameras.height, cameras.height], dim=0),
)
self.train_dataset = TrivialDataset(cameras, image)
self.eval_dataset = TrivialDataset(cameras, image)
self.pixel_sampler = PixelSampler(self.config.train_resolution**2)
# pylint: disable=non-parent-init-called
DataManager.__init__(self)
if self.config.use_input_transient:
self.input_mask = Parameter(torch.ones(image.shape[:2]).float(), requires_grad=True)
def next_input(self, step: int) -> Tuple[RayBundle, Dict]:
"""Returns the batch for the input image"""
if self.input_image_mask is None:
batch = self.pixel_sampler.sample({"image": self.input_image.unsqueeze(0), "image_idx": torch.tensor([0])})
else:
batch = self.pixel_sampler.sample(
{"image": self.input_image.unsqueeze(0), "image_idx": torch.tensor([0]), "mask": self.input_image_mask}
)
batch["input_image"] = True
ray_indices = batch["indices"]
c = ray_indices[:, 0] # camera indices
y = ray_indices[:, 1] # row indices
x = ray_indices[:, 2] # col indices
coords = self.original_cameras.get_image_coords()[y, x]
if self.config.use_input_transient:
input_mask_pixels = torch.sigmoid(self.input_mask[y, x])
batch["input_mask"] = input_mask_pixels
original_camera = self.original_cameras
if self.config.jitter_std:
original_camera = copy.deepcopy(original_camera)
original_camera.camera_to_worlds = centralized_jitter(
original_camera.camera_to_worlds, self.config.input_image_position_jitter_std * min(step, 300) / 300
)
ray_bundle = original_camera.generate_rays(
camera_indices=c.unsqueeze(-1),
coords=coords,
)
return ray_bundle, batch
def next_train(self, step: int) -> Tuple[RayBundle, Dict]:
"""Returns the next batch of data from the train dataloader."""
self.train_count += 1
# randomly choose a random camera view or the input view here
if np.random.random_sample() < 0.1:
return self.next_input(step)
# # TODO Reimplement when cameras are fully working
# if step > 2000:
# cameras, _, _ = random_train_pose(
# self.config.train_images_per_batch, self.config.train_resolution, device=self.device
# )
# ray_bundle = cameras.generate_rays(
# torch.tensor(list(range(self.config.train_images_per_batch)))
# ).flatten()
# return ray_bundle, {"initialization": False}
# TODO below
if step < self.config.anneal_camera_angles and self.config.anneal_camera_angles > 0:
cameras, vertical_rotation, central_rotation = random_train_pose(
self.config.train_images_per_batch,
self.config.train_resolution,
device=self.device,
radius_mean=self.config.radius_mean,
radius_std=self.config.radius_std,
focal_range=self.config.focal_range,
vertical_rotation_range=torch.tensor(self.config.vertical_rotation_range)
* step
/ (self.config.anneal_camera_angles),
central_rotation_range=torch.tensor(self.config.central_rotation_range)
* step
/ (self.config.anneal_camera_angles),
jitter_std=self.config.jitter_std,
)
else:
cameras, vertical_rotation, central_rotation = random_train_pose(
self.config.train_images_per_batch,
self.config.train_resolution,
device=self.device,
radius_mean=self.config.radius_mean,
radius_std=self.config.radius_std,
focal_range=self.config.focal_range,
vertical_rotation_range=self.config.vertical_rotation_range,
central_rotation_range=self.config.central_rotation_range,
jitter_std=self.config.jitter_std,
)
ray_bundle = cameras.generate_rays(torch.tensor(list(range(self.config.train_images_per_batch)))).flatten()
# camera_idx = torch.randint(0, self.eval_cameras.shape[0], [1], dtype=torch.long, device=self.device)
# ray_bundle = self.eval_cameras.generate_rays(camera_idx).flatten()
return ray_bundle, {"vertical": vertical_rotation, "central": central_rotation, "initialization": True, "input_image": False}
def get_param_groups(self) -> Dict[str, List[Parameter]]: # pylint: disable=no-self-use
"""Get the param groups for the data manager.
Returns:
A list of dictionaries containing the data manager's param groups.
"""
if self.config.use_input_transient:
param_groups = {"transients": [self.input_mask]}
else:
return {}
return param_groups
def setup_train(self):
pass
def setup_eval(self):
pass
@@ -12,18 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
<<<<<<< HEAD
"""Utility functions for Stable Diffusion"""
def get_text_embedding(batch, prompting_type, prompts, device):
"""util function for getting correct text_embedding from view direction
Args:
batch: contains ray batch information
prompting_type: how to sample text embeddings
prompts: dictionary of text embeddings
"""
=======
"""
Stable diffusion utils
"""
@@ -37,41 +25,10 @@ def get_text_embedding(batch, prompting_type, text_embeddings, sd_device):
top = text_embeddings["top_text_embedding"]
back = text_embeddings["back_text_embedding"]
>>>>>>> a474aa3fb81d07d55beb27685b4e8d6cd5bf9514
assert prompting_type in ("location_based", "interpolated", "base")
if prompting_type == "location_based":
if batch["vertical"] < 40:
<<<<<<< HEAD
text_embedding = prompts["top_text_embedding"]
elif batch["central"] > 315 or batch["central"] <= 45:
text_embedding = prompts["front_text_embedding"]
elif batch["central"] > 45 and batch["central"] <= 135:
text_embedding = prompts["side_text_embedding"]
elif batch["central"] > 135 and batch["central"] <= 225:
text_embedding = prompts["back_text_embedding"]
else: # batch["central"] > 225 and batch["central"] <= 315:
text_embedding = prompts["side_text_embedding"]
elif prompting_type == "interpolated":
horiz = batch["central"].to(device)
vert = max(batch["vertical"].to(device), 0)
if batch["central"] > 0 or batch["central"] <= 90:
text_embedding = (horiz) * prompts["side_text_embedding"] + (90 - horiz) * prompts["front_text_embedding"]
elif batch["central"] > 90 and batch["central"] <= 180:
text_embedding = (horiz - 90) * prompts["back_text_embedding"] + (180 - horiz) * prompts["side_text_embedding"]
elif batch["central"] > 180 and batch["central"] <= 270:
text_embedding = (horiz - 180) * prompts["side_text_embedding"] + (270 - horiz) * prompts["back_text_embedding"]
else: # batch["central"] > 270 and batch["central"] <= 360:
text_embedding = (horiz - 270) * prompts["front_text_embedding"] + (360 - horiz) * prompts["side_text_embedding"]
text_embedding = text_embedding / 90.0
text_embedding = (vert * text_embedding + (90 - vert) * prompts["top_text_embedding"]) / 90.0
else:
text_embedding = prompts["base_text_embedding"]
=======
text_embedding = top
elif batch["central"] > 315 or batch["central"] <= 45:
text_embedding = front
@@ -100,6 +57,5 @@ def get_text_embedding(batch, prompting_type, text_embeddings, sd_device):
elif prompting_type == "base":
text_embedding = base
>>>>>>> a474aa3fb81d07d55beb27685b4e8d6cd5bf9514
return text_embedding
+227
View File
@@ -0,0 +1,227 @@
# Copyright 2022 The Nerfstudio Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
DreamFusion implementation.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Dict, List, Tuple, Type, Optional
import numpy as np
import torch
from torch.nn import Parameter
from torch.cuda.amp.grad_scaler import GradScaler
from nerfstudio.cameras.rays import RayBundle
from nerfstudio.engine.callbacks import (
TrainingCallback,
TrainingCallbackAttributes,
TrainingCallbackLocation,
)
from nerfstudio.generative.stable_diffusion import StableDiffusion
from nerfstudio.generative.stable_diffusion_utils import get_text_embedding
from nerfstudio.field_components.field_heads import FieldHeadNames
from nerfstudio.fields.density_fields import HashMLPDensityField
from nerfstudio.fields.dreamfusion_field import DreamFusionField
from nerfstudio.model_components.losses import (
MSELoss,
distortion_loss,
interlevel_loss,
orientation_loss,
pred_normal_loss,
)
from nerfstudio.model_components.ray_samplers import (
ProposalNetworkSampler,
UniformSampler,
)
from nerfstudio.model_components.renderers import (
AccumulationRenderer,
DepthRenderer,
NormalsRenderer,
RGBRenderer,
)
from nerfstudio.model_components.scene_colliders import AABBBoxCollider, SphereCollider
from nerfstudio.model_components.shaders import LambertianShader, NormalsShader
from nerfstudio.models.base_model import Model, ModelConfig
from nerfstudio.models.dreamfusion import DreamFusionModel, DreamFusionModelConfig
from nerfstudio.utils import colormaps, colors, math, misc
@dataclass
class DreamFusionPlusModelConfig(DreamFusionModelConfig):
"""DreamFusion model config"""
_target: Type = field(default_factory=lambda: DreamFusionPlusModel)
"""target class to instantiate"""
prompt: str = "A high-quality photo of a tree frog on a stump"
"""prompt for stable dreamfusion"""
orientation_loss_mult: float = 0.0001
"""Orientation loss multipier on computed normals."""
pred_normal_loss_mult: float = 0.001
"""Predicted normal loss multiplier."""
random_light_source: bool = True
"""Randomizes light source per output."""
initialize_density: bool = True
"""Initialize density in center of scene."""
init_density_strength: float = 0.1
"""Initial strength of center density"""
sphere_collider: bool = True
"""Use spherical collider instead of box"""
random_background: bool = True
"""Randomly choose between using background mlp and random color for background"""
target_transmittance_start: float = 0.4
"""target transmittance for opacity penalty. This is the percent of the scene that is
background when rendered at the start of training"""
target_transmittance_end: float = 0.7
"""target transmittance for opacity penalty. This is the percent of the scene that is
background when rendered at the end of training"""
transmittance_end_schedule: int = 1500
"""number of iterations to reach target_transmittance_end"""
num_proposal_samples_per_ray: Tuple[int, ...] = (256, 96)
"""Number of samples per ray for each proposal network."""
num_nerf_samples_per_ray: int = 48
"""Number of samples per ray for the nerf network."""
proposal_update_every: int = 5
"""Sample every n steps after the warmup"""
proposal_warmup: int = 5000
"""Scales n from 1 to proposal_update_every over this many steps"""
num_proposal_iterations: int = 2
"""Number of proposal network iterations."""
use_same_proposal_network: bool = False
"""Use the same proposal network. Otherwise use different ones."""
proposal_net_args_list: List[Dict] = field(
default_factory=lambda: [
{"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 128},
{"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 256},
]
)
"""Arguments for the proposal density fields."""
proposal_weights_anneal_slope: float = 10.0
"""Slope of the annealing function for the proposal weights."""
proposal_weights_anneal_max_num_iters: int = 500
"""Max num iterations for the annealing function."""
use_single_jitter: bool = True
"""Whether use single jitter or not for the proposal networks."""
interlevel_loss_mult: float = 1.0
"""Proposal loss multiplier."""
distortion_loss_mult: float = 1.0
"""Distortion loss multiplier."""
start_normals_training: int = 1000
"""Start training normals after this many iterations"""
start_lambertian_training: int = 1000
"""start training with lambertian shading after this many iterations"""
opacity_penalty: bool = True
"""enables penalty to encourage sparse weights (penalizing for uniform density along ray)"""
opacity_loss_mult: float = 1
"""scale for opacity penalty"""
max_res: int = 256
"""Maximum resolution of the density field."""
prompting_type: str = "location_based"
"""choose between location_based, interpolated, base"""
top_prompt: str = ", overhead view"
"""appended to prompt for overhead view"""
side_prompt: str = ", side view"
"""appended to prompt for side view"""
front_prompt: str = ", front view"
"""appended to prompt for front view"""
back_prompt: str = ", back view"
"""appended to prompt for back view"""
guidance_scale: float = 100
"""guidance scale for sds loss"""
stablediffusion_device: Optional[str] = None
"""device for stable diffusion"""
sd_version: str = '1-5'
class DreamFusionPlusModel(DreamFusionModel):
"""DreamFusionModel Model
Args:
config: DreamFusion configuration to instantiate model
"""
config: DreamFusionPlusModelConfig
def __init__(
self,
config: DreamFusionPlusModelConfig,
**kwargs,
) -> None:
super().__init__(config=config, **kwargs)
def get_loss_dict(self, outputs, batch, metrics_dict=None) -> Dict[str, torch.Tensor]:
# Scaling metrics by coefficients to create the losses.
loss_dict = {}
loss_dict = misc.scale_dict(loss_dict, self.config.loss_coefficients)
if self.train_normals:
# orientation loss for computed normals
loss_dict["orientation_loss"] = self.config.orientation_loss_mult * torch.mean(
outputs["rendered_orientation_loss"]
)
# ground truth supervision for normals
loss_dict["pred_normal_loss"] = self.config.pred_normal_loss_mult * torch.mean(
outputs["rendered_pred_normal_loss"]
)
else:
loss_dict["orientation_loss"] = 0
loss_dict["pred_normal_loss"] = 0
if self.config.opacity_penalty:
loss_dict["opacity_loss"] = self.config.opacity_loss_mult * outputs["opacity_loss"].mean()
if self.prompt != self.cur_prompt:
self.cur_prompt = self.prompt
self.text_embeddings = {
"top_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.top_prompt}", ""),
"front_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.front_prompt}", ""),
"side_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.side_prompt}", ""),
"back_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.back_prompt}", ""),
"base_text_embedding": self.sd.get_text_embeds(self.cur_prompt, "")
}
if batch["input_image"]:
loss_dict["rgb_loss"] = self.rgb_loss(batch["image"].to(self.device), outputs["rgb"])
else:
text_embedding = get_text_embedding(batch, self.prompting_type, self.text_embeddings, self.sd_device)
train_output = (
outputs["train_output"].view(1, int(outputs["train_output"].shape[0] ** 0.5), int(outputs["train_output"].shape[0] ** 0.5), 3)
.permute(0, 3, 1, 2)
)
with torch.autocast(device_type="cuda", dtype=torch.float16):
sds_loss = self.sd.sds_loss(
text_embedding.to(self.sd_device),
train_output.to(self.sd_device),
guidance_scale=int(self.guidance_scale),
grad_scaler=self.grad_scaler,
)
loss_dict["sds_loss"] = sds_loss.to(self.device)
if self.training:
loss_dict["distortion_loss"] = self.config.distortion_loss_mult * distortion_loss(
outputs["weights_list"], outputs["ray_samples_list"]
)
loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss(
outputs["weights_list"], outputs["ray_samples_list"]
)
return loss_dict