From 0a4d49c4a457cb7814fc033f246aea3f1201342b Mon Sep 17 00:00:00 2001 From: terrance Date: Tue, 21 Feb 2023 20:23:05 +0000 Subject: [PATCH] df plus --- nerfstudio/configs/method_configs.py | 49 +++ .../dreamfusionplus_datamanager.py | 338 ++++++++++++++++++ .../generative/stable_diffusion_utils.py | 44 --- nerfstudio/models/dreamfusionplus.py | 227 ++++++++++++ 4 files changed, 614 insertions(+), 44 deletions(-) create mode 100644 nerfstudio/data/datamanagers/dreamfusionplus_datamanager.py create mode 100644 nerfstudio/models/dreamfusionplus.py diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index dc9aba07f..b56f27cc8 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -30,6 +30,7 @@ from nerfstudio.data.datamanagers.depth_datamanager import DepthDataManagerConfi from nerfstudio.data.datamanagers.dreamfusion_datamanager import ( DreamFusionDataManagerConfig, ) +from nerfstudio.data.datamanagers.dreamfusionplus_datamanager import DreamFusionPlusDataManagerConfig from nerfstudio.data.datamanagers.semantic_datamanager import SemanticDataManagerConfig from nerfstudio.data.datamanagers.variable_res_datamanager import ( VariableResDataManagerConfig, @@ -51,6 +52,7 @@ from nerfstudio.engine.trainer import TrainerConfig from nerfstudio.field_components.temporal_distortions import TemporalDistortionKind from nerfstudio.models.depth_nerfacto import DepthNerfactoModelConfig from nerfstudio.models.dreamfusion import DreamFusionModelConfig +from nerfstudio.models.dreamfusionplus import DreamFusionPlusModelConfig from nerfstudio.models.instant_ngp import InstantNGPModelConfig from nerfstudio.models.mipnerf import MipNerfModel from nerfstudio.models.nerfacto import NerfactoModelConfig @@ -383,6 +385,53 @@ method_configs["dreamfusion"] = TrainerConfig( vis="viewer", ) +method_configs["dreamplus"] = TrainerConfig( + method_name="dreamplus", + steps_per_eval_batch=50, + steps_per_eval_image=50, + steps_per_save=200, + max_num_iterations=30000, + mixed_precision=True, + pipeline=VanillaPipelineConfig( + generative=True, + datamanager=DreamFusionPlusDataManagerConfig( + dataparser=NerfstudioDataParserConfig(), + train_num_rays_per_batch=4096, + eval_num_rays_per_batch=4096, + ), + model=DreamFusionPlusModelConfig( + eval_num_rays_per_chunk=1 << 15, + distortion_loss_mult=0.02, + interlevel_loss_mult=1.0, + orientation_loss_mult=0.1, + max_res=256, + sphere_collider=True, + initialize_density=False, + random_background=True, + proposal_warmup=1000, + proposal_update_every=2, + proposal_weights_anneal_max_num_iters=100, + start_lambertian_training=1000, + start_normals_training=500, + opacity_loss_mult=0.001, + prompting_type="location_based", + guidance_scale=100, + ), + ), + optimizers={ + "proposal_networks": { + "optimizer": AdamOptimizerConfig(lr=5e-2, eps=1e-15), + "scheduler": None, + }, + "fields": { + "optimizer": AdamOptimizerConfig(lr=5e-3, eps=1e-15), + "scheduler": None, + }, + }, + viewer=ViewerConfig(num_rays_per_chunk=1 << 15), + vis="viewer", +) + method_configs["nerfplayer-nerfacto"] = TrainerConfig( method_name="nerfplayer-nerfacto", pipeline=VanillaPipelineConfig( diff --git a/nerfstudio/data/datamanagers/dreamfusionplus_datamanager.py b/nerfstudio/data/datamanagers/dreamfusionplus_datamanager.py new file mode 100644 index 000000000..7797b77c8 --- /dev/null +++ b/nerfstudio/data/datamanagers/dreamfusionplus_datamanager.py @@ -0,0 +1,338 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Data manager for dreamfusion +""" + +from __future__ import annotations + +import copy +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, List, Optional, Tuple, Type, Union + +import numpy as np +import torch +from PIL import Image +from rich.progress import Console +from torch.nn import Parameter +from typing_extensions import Literal + +from nerfstudio.cameras.cameras import Cameras +from nerfstudio.cameras.rays import RayBundle +from nerfstudio.configs.config_utils import to_immutable_dict +from nerfstudio.data.datamanagers.base_datamanager import DataManager +from nerfstudio.data.datamanagers.dreamfusion_datamanager import ( + DreamFusionDataManager, + DreamFusionDataManagerConfig, + random_train_pose, +) +from nerfstudio.data.datasets.base_dataset import InputDataset +from nerfstudio.data.pixel_samplers import PixelSampler +from nerfstudio.data.scene_box import SceneBox + +CONSOLE = Console(width=120) + + +def centralized_jitter(c2w, jitter_std): + assert c2w.shape[0] == 1 + theta = torch.randn(1) * jitter_std + phi = torch.randn(1) * jitter_std + R = torch.Tensor( + [ + [torch.cos(theta), torch.sin(theta), 0], + [-torch.sin(theta), torch.cos(theta), 0], + [0, 0, 1], + ] + ) + R = R @ torch.Tensor( + [ + [1, 0, 0], + [0, torch.cos(phi), -torch.sin(phi)], + [0, torch.sin(phi), torch.cos(phi)], + ] + ) + + R = R.to(c2w.device) + + jittered = copy.deepcopy(c2w) + + jittered[..., :3, :3] = R @ jittered[..., :3, :3] + jittered[..., :3, 3] = (R @ jittered[..., :3, 3].unsqueeze(-1)).squeeze(-1) + + return jittered + + +class TrivialDataset(InputDataset): + """A trivial dataset with blank images for the viewer""" + + # pylint: disable=super-init-not-called + def __init__(self, cameras: Cameras, input_img: torch.Tensor): + self.size = cameras.size + self.cameras = cameras + self.alpha_color = None + self.scene_box = SceneBox(torch.Tensor([[-1, -1, -1], [1, 1, 1]])) + self.mask_filenames = None + self.metadata = to_immutable_dict({}) + self.input_img = input_img + + def __len__(self): + return self.size + + def __getitem__(self, index: int) -> Dict: + if index == 0: + return {"image": self.input_img, "image_idx": index} + return {"image": torch.cat([torch.ones(128, 256, 3), torch.zeros(128, 256, 3)], dim=0), "image_idx": index} + + +@dataclass +class DreamFusionPlusDataManagerConfig(DreamFusionDataManagerConfig): + """Configuration for data manager instantiation; DataManager is in charge of keeping the train/eval dataparsers; + After instantiation, data manager holds both train/eval datasets and is in charge of returning unpacked + train/eval data at each iteration + """ + + _target: Type = field(default_factory=lambda: DreamFusionPlusDataManager) + input_image_path: Path = Path("data/generative/chair/chair.jpg") + """Directory specifying location of input image.""" + input_image_mask_path: Optional[Path] = None + """Directory specifying location of input image mask, if one exists""" + radius_mean: float = 2.2 + """Mean radius of the camera sphere""" + use_input_transient: bool = False + """Whether to allow the input image to mask out parts of the image that give poor rgb loss""" + input_distance: float = 1.5 + """Scalar multiple of radius_mean dictating how close we think this image was taken to the object in question. + Large values mean this image was taken relatively far away, small values mean this was taken from relatively + close to the object""" + input_image_position_jitter_std: float = 0 + """Whether or not to jitter the position of the input image each time to prevent accumulation directly + in front of the camera. If this is nonozero, it is the standard deviation of the jitter""" + reference_directory: Path = Path("") + """Directory specifying location of reference images.""" + anneal_camera_angles: float = 0 + """number of steps to anneal camera angles. We start near the input image and slowly work our way outwards + as we train with this greater than 0""" + central_rotation_range: Tuple[float, float] = (-180, 180) + """Range of central rotation""" + + +class DreamFusionPlusDataManager(DreamFusionDataManager): # pylint: disable=abstract-method + """Basic stored data manager implementation. + This is pretty much a port over from our old dataloading utilities, and is a little jank + under the hood. We may clean this up a little bit under the hood with more standard dataloading + components that can be strung together, but it can be just used as a black box for now since + only the constructor is likely to change in the future, or maybe passing in step number to the + next_train and next_eval functions. + Args: + config: the DataManagerConfig used to instantiate class + """ + + config: DreamFusionPlusDataManagerConfig + + # pylint: disable=super-init-not-called + def __init__( + self, + config: DreamFusionPlusDataManagerConfig, + device: Union[torch.device, str] = "cpu", + test_mode: Literal["test", "val", "inference"] = "val", + world_size: int = 1, + local_rank: int = 0, + **kwargs, # pylint: disable=unused-argument + ): + self.config = config + self.device = device + self.world_size = world_size + self.local_rank = local_rank + self.sampler = None + self.test_mode = test_mode + self.test_split = "test" if test_mode in ["test", "inference"] else "val" + self.dataparser = self.config.dataparser.setup() + + pil_image = Image.open(self.config.input_image_path) + image = np.array(pil_image, dtype="uint8") # shape is (h, w) or (h, w, 3 or 4) + if len(image.shape) == 2: + image = image[:, :, None].repeat(3, axis=2) + assert image.shape[-1] == 3, "Image must be RGB" + image = torch.from_numpy(image).to(self.device).float() / 255.0 + self.input_image = image.to(device) + self.input_image_mask = None + if self.config.input_image_mask_path is not None: + pil_mask = Image.open(self.config.input_image_mask_path) + mask = torch.from_numpy(np.array(pil_mask)) + self.input_image_mask = torch.any(mask != 0, dim=-1) + + # Transients for the input image, allows it to mask out parts of the image that are background + + original_cameras, _, _ = random_train_pose( + 1, + 1, + device=torch.device("cpu"), + radius_mean=self.config.radius_mean * self.config.input_distance, + radius_std=0, + focal_range=[1, 1], + vertical_rotation_range=[0, 0], + central_rotation_range=[0, 0], + jitter_std=0, + ) + + original_cameras = Cameras( + camera_to_worlds=original_cameras.camera_to_worlds, + fx=float(image.shape[1]), + fy=float(image.shape[0]), + cx=image.shape[1] / 2, + cy=image.shape[0] / 2, + ) + self.original_cameras = original_cameras.to(self.device) + + cameras, _, _ = random_train_pose( + self.config.num_eval_angles, + self.config.eval_resolution, + device=self.device, + radius_mean=self.config.radius_mean, + radius_std=self.config.radius_std, + focal_range=self.config.focal_range, + vertical_rotation_range=self.config.vertical_rotation_range, + central_rotation_range=self.config.central_rotation_range, + jitter_std=self.config.jitter_std, + ) + cameras = cameras.to("cpu") + + cameras = Cameras( + camera_to_worlds=torch.cat([original_cameras.camera_to_worlds, cameras.camera_to_worlds], dim=0), + fx=torch.cat([original_cameras.fx, cameras.fx], dim=0), + fy=torch.cat([original_cameras.fy, cameras.fy], dim=0), + cx=torch.cat([original_cameras.cx, cameras.cx], dim=0), + cy=torch.cat([original_cameras.cy, cameras.cy], dim=0), + width=torch.cat([original_cameras.width, cameras.width], dim=0), + height=torch.cat([original_cameras.height, cameras.height], dim=0), + ) + + self.train_dataset = TrivialDataset(cameras, image) + self.eval_dataset = TrivialDataset(cameras, image) + + self.pixel_sampler = PixelSampler(self.config.train_resolution**2) + + # pylint: disable=non-parent-init-called + DataManager.__init__(self) + if self.config.use_input_transient: + self.input_mask = Parameter(torch.ones(image.shape[:2]).float(), requires_grad=True) + + def next_input(self, step: int) -> Tuple[RayBundle, Dict]: + """Returns the batch for the input image""" + + if self.input_image_mask is None: + batch = self.pixel_sampler.sample({"image": self.input_image.unsqueeze(0), "image_idx": torch.tensor([0])}) + else: + batch = self.pixel_sampler.sample( + {"image": self.input_image.unsqueeze(0), "image_idx": torch.tensor([0]), "mask": self.input_image_mask} + ) + + batch["input_image"] = True + + ray_indices = batch["indices"] + c = ray_indices[:, 0] # camera indices + y = ray_indices[:, 1] # row indices + x = ray_indices[:, 2] # col indices + coords = self.original_cameras.get_image_coords()[y, x] + if self.config.use_input_transient: + input_mask_pixels = torch.sigmoid(self.input_mask[y, x]) + batch["input_mask"] = input_mask_pixels + + original_camera = self.original_cameras + if self.config.jitter_std: + original_camera = copy.deepcopy(original_camera) + original_camera.camera_to_worlds = centralized_jitter( + original_camera.camera_to_worlds, self.config.input_image_position_jitter_std * min(step, 300) / 300 + ) + ray_bundle = original_camera.generate_rays( + camera_indices=c.unsqueeze(-1), + coords=coords, + ) + return ray_bundle, batch + + def next_train(self, step: int) -> Tuple[RayBundle, Dict]: + """Returns the next batch of data from the train dataloader.""" + + self.train_count += 1 + + # randomly choose a random camera view or the input view here + if np.random.random_sample() < 0.1: + return self.next_input(step) + + # # TODO Reimplement when cameras are fully working + # if step > 2000: + # cameras, _, _ = random_train_pose( + # self.config.train_images_per_batch, self.config.train_resolution, device=self.device + # ) + + # ray_bundle = cameras.generate_rays( + # torch.tensor(list(range(self.config.train_images_per_batch))) + # ).flatten() + # return ray_bundle, {"initialization": False} + + # TODO below + if step < self.config.anneal_camera_angles and self.config.anneal_camera_angles > 0: + cameras, vertical_rotation, central_rotation = random_train_pose( + self.config.train_images_per_batch, + self.config.train_resolution, + device=self.device, + radius_mean=self.config.radius_mean, + radius_std=self.config.radius_std, + focal_range=self.config.focal_range, + vertical_rotation_range=torch.tensor(self.config.vertical_rotation_range) + * step + / (self.config.anneal_camera_angles), + central_rotation_range=torch.tensor(self.config.central_rotation_range) + * step + / (self.config.anneal_camera_angles), + jitter_std=self.config.jitter_std, + ) + else: + cameras, vertical_rotation, central_rotation = random_train_pose( + self.config.train_images_per_batch, + self.config.train_resolution, + device=self.device, + radius_mean=self.config.radius_mean, + radius_std=self.config.radius_std, + focal_range=self.config.focal_range, + vertical_rotation_range=self.config.vertical_rotation_range, + central_rotation_range=self.config.central_rotation_range, + jitter_std=self.config.jitter_std, + ) + ray_bundle = cameras.generate_rays(torch.tensor(list(range(self.config.train_images_per_batch)))).flatten() + + # camera_idx = torch.randint(0, self.eval_cameras.shape[0], [1], dtype=torch.long, device=self.device) + # ray_bundle = self.eval_cameras.generate_rays(camera_idx).flatten() + + return ray_bundle, {"vertical": vertical_rotation, "central": central_rotation, "initialization": True, "input_image": False} + + def get_param_groups(self) -> Dict[str, List[Parameter]]: # pylint: disable=no-self-use + """Get the param groups for the data manager. + Returns: + A list of dictionaries containing the data manager's param groups. + """ + if self.config.use_input_transient: + param_groups = {"transients": [self.input_mask]} + else: + return {} + + return param_groups + + def setup_train(self): + pass + + def setup_eval(self): + pass \ No newline at end of file diff --git a/nerfstudio/generative/stable_diffusion_utils.py b/nerfstudio/generative/stable_diffusion_utils.py index a02648cf4..82f61ff7a 100644 --- a/nerfstudio/generative/stable_diffusion_utils.py +++ b/nerfstudio/generative/stable_diffusion_utils.py @@ -12,18 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -<<<<<<< HEAD -"""Utility functions for Stable Diffusion""" - -def get_text_embedding(batch, prompting_type, prompts, device): - """util function for getting correct text_embedding from view direction - Args: - batch: contains ray batch information - prompting_type: how to sample text embeddings - prompts: dictionary of text embeddings - - """ -======= """ Stable diffusion utils """ @@ -37,41 +25,10 @@ def get_text_embedding(batch, prompting_type, text_embeddings, sd_device): top = text_embeddings["top_text_embedding"] back = text_embeddings["back_text_embedding"] ->>>>>>> a474aa3fb81d07d55beb27685b4e8d6cd5bf9514 assert prompting_type in ("location_based", "interpolated", "base") if prompting_type == "location_based": if batch["vertical"] < 40: -<<<<<<< HEAD - text_embedding = prompts["top_text_embedding"] - elif batch["central"] > 315 or batch["central"] <= 45: - text_embedding = prompts["front_text_embedding"] - elif batch["central"] > 45 and batch["central"] <= 135: - text_embedding = prompts["side_text_embedding"] - elif batch["central"] > 135 and batch["central"] <= 225: - text_embedding = prompts["back_text_embedding"] - else: # batch["central"] > 225 and batch["central"] <= 315: - text_embedding = prompts["side_text_embedding"] - - elif prompting_type == "interpolated": - horiz = batch["central"].to(device) - vert = max(batch["vertical"].to(device), 0) - - if batch["central"] > 0 or batch["central"] <= 90: - text_embedding = (horiz) * prompts["side_text_embedding"] + (90 - horiz) * prompts["front_text_embedding"] - elif batch["central"] > 90 and batch["central"] <= 180: - text_embedding = (horiz - 90) * prompts["back_text_embedding"] + (180 - horiz) * prompts["side_text_embedding"] - elif batch["central"] > 180 and batch["central"] <= 270: - text_embedding = (horiz - 180) * prompts["side_text_embedding"] + (270 - horiz) * prompts["back_text_embedding"] - else: # batch["central"] > 270 and batch["central"] <= 360: - text_embedding = (horiz - 270) * prompts["front_text_embedding"] + (360 - horiz) * prompts["side_text_embedding"] - - text_embedding = text_embedding / 90.0 - text_embedding = (vert * text_embedding + (90 - vert) * prompts["top_text_embedding"]) / 90.0 - - else: - text_embedding = prompts["base_text_embedding"] -======= text_embedding = top elif batch["central"] > 315 or batch["central"] <= 45: text_embedding = front @@ -100,6 +57,5 @@ def get_text_embedding(batch, prompting_type, text_embeddings, sd_device): elif prompting_type == "base": text_embedding = base ->>>>>>> a474aa3fb81d07d55beb27685b4e8d6cd5bf9514 return text_embedding diff --git a/nerfstudio/models/dreamfusionplus.py b/nerfstudio/models/dreamfusionplus.py new file mode 100644 index 000000000..1c7b9eed4 --- /dev/null +++ b/nerfstudio/models/dreamfusionplus.py @@ -0,0 +1,227 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +DreamFusion implementation. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict, List, Tuple, Type, Optional + +import numpy as np +import torch +from torch.nn import Parameter +from torch.cuda.amp.grad_scaler import GradScaler + +from nerfstudio.cameras.rays import RayBundle +from nerfstudio.engine.callbacks import ( + TrainingCallback, + TrainingCallbackAttributes, + TrainingCallbackLocation, +) +from nerfstudio.generative.stable_diffusion import StableDiffusion +from nerfstudio.generative.stable_diffusion_utils import get_text_embedding +from nerfstudio.field_components.field_heads import FieldHeadNames +from nerfstudio.fields.density_fields import HashMLPDensityField +from nerfstudio.fields.dreamfusion_field import DreamFusionField +from nerfstudio.model_components.losses import ( + MSELoss, + distortion_loss, + interlevel_loss, + orientation_loss, + pred_normal_loss, +) +from nerfstudio.model_components.ray_samplers import ( + ProposalNetworkSampler, + UniformSampler, +) +from nerfstudio.model_components.renderers import ( + AccumulationRenderer, + DepthRenderer, + NormalsRenderer, + RGBRenderer, +) +from nerfstudio.model_components.scene_colliders import AABBBoxCollider, SphereCollider +from nerfstudio.model_components.shaders import LambertianShader, NormalsShader +from nerfstudio.models.base_model import Model, ModelConfig +from nerfstudio.models.dreamfusion import DreamFusionModel, DreamFusionModelConfig +from nerfstudio.utils import colormaps, colors, math, misc + + +@dataclass +class DreamFusionPlusModelConfig(DreamFusionModelConfig): + """DreamFusion model config""" + + _target: Type = field(default_factory=lambda: DreamFusionPlusModel) + """target class to instantiate""" + prompt: str = "A high-quality photo of a tree frog on a stump" + """prompt for stable dreamfusion""" + + orientation_loss_mult: float = 0.0001 + """Orientation loss multipier on computed normals.""" + pred_normal_loss_mult: float = 0.001 + """Predicted normal loss multiplier.""" + random_light_source: bool = True + """Randomizes light source per output.""" + initialize_density: bool = True + """Initialize density in center of scene.""" + init_density_strength: float = 0.1 + """Initial strength of center density""" + sphere_collider: bool = True + """Use spherical collider instead of box""" + random_background: bool = True + """Randomly choose between using background mlp and random color for background""" + target_transmittance_start: float = 0.4 + """target transmittance for opacity penalty. This is the percent of the scene that is + background when rendered at the start of training""" + target_transmittance_end: float = 0.7 + """target transmittance for opacity penalty. This is the percent of the scene that is + background when rendered at the end of training""" + transmittance_end_schedule: int = 1500 + """number of iterations to reach target_transmittance_end""" + num_proposal_samples_per_ray: Tuple[int, ...] = (256, 96) + """Number of samples per ray for each proposal network.""" + num_nerf_samples_per_ray: int = 48 + """Number of samples per ray for the nerf network.""" + proposal_update_every: int = 5 + """Sample every n steps after the warmup""" + proposal_warmup: int = 5000 + """Scales n from 1 to proposal_update_every over this many steps""" + num_proposal_iterations: int = 2 + """Number of proposal network iterations.""" + use_same_proposal_network: bool = False + """Use the same proposal network. Otherwise use different ones.""" + proposal_net_args_list: List[Dict] = field( + default_factory=lambda: [ + {"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 128}, + {"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 256}, + ] + ) + """Arguments for the proposal density fields.""" + proposal_weights_anneal_slope: float = 10.0 + """Slope of the annealing function for the proposal weights.""" + proposal_weights_anneal_max_num_iters: int = 500 + """Max num iterations for the annealing function.""" + use_single_jitter: bool = True + """Whether use single jitter or not for the proposal networks.""" + interlevel_loss_mult: float = 1.0 + """Proposal loss multiplier.""" + distortion_loss_mult: float = 1.0 + """Distortion loss multiplier.""" + start_normals_training: int = 1000 + """Start training normals after this many iterations""" + start_lambertian_training: int = 1000 + """start training with lambertian shading after this many iterations""" + opacity_penalty: bool = True + """enables penalty to encourage sparse weights (penalizing for uniform density along ray)""" + opacity_loss_mult: float = 1 + """scale for opacity penalty""" + max_res: int = 256 + """Maximum resolution of the density field.""" + + prompting_type: str = "location_based" + """choose between location_based, interpolated, base""" + top_prompt: str = ", overhead view" + """appended to prompt for overhead view""" + side_prompt: str = ", side view" + """appended to prompt for side view""" + front_prompt: str = ", front view" + """appended to prompt for front view""" + back_prompt: str = ", back view" + """appended to prompt for back view""" + guidance_scale: float = 100 + """guidance scale for sds loss""" + stablediffusion_device: Optional[str] = None + """device for stable diffusion""" + sd_version: str = '1-5' + + +class DreamFusionPlusModel(DreamFusionModel): + """DreamFusionModel Model + + Args: + config: DreamFusion configuration to instantiate model + """ + + config: DreamFusionPlusModelConfig + + def __init__( + self, + config: DreamFusionPlusModelConfig, + **kwargs, + ) -> None: + super().__init__(config=config, **kwargs) + + + def get_loss_dict(self, outputs, batch, metrics_dict=None) -> Dict[str, torch.Tensor]: + # Scaling metrics by coefficients to create the losses. + + loss_dict = {} + loss_dict = misc.scale_dict(loss_dict, self.config.loss_coefficients) + if self.train_normals: + # orientation loss for computed normals + loss_dict["orientation_loss"] = self.config.orientation_loss_mult * torch.mean( + outputs["rendered_orientation_loss"] + ) + # ground truth supervision for normals + loss_dict["pred_normal_loss"] = self.config.pred_normal_loss_mult * torch.mean( + outputs["rendered_pred_normal_loss"] + ) + else: + loss_dict["orientation_loss"] = 0 + loss_dict["pred_normal_loss"] = 0 + + if self.config.opacity_penalty: + loss_dict["opacity_loss"] = self.config.opacity_loss_mult * outputs["opacity_loss"].mean() + + if self.prompt != self.cur_prompt: + self.cur_prompt = self.prompt + self.text_embeddings = { + "top_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.top_prompt}", ""), + "front_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.front_prompt}", ""), + "side_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.side_prompt}", ""), + "back_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.back_prompt}", ""), + "base_text_embedding": self.sd.get_text_embeds(self.cur_prompt, "") + } + + if batch["input_image"]: + loss_dict["rgb_loss"] = self.rgb_loss(batch["image"].to(self.device), outputs["rgb"]) + else: + text_embedding = get_text_embedding(batch, self.prompting_type, self.text_embeddings, self.sd_device) + + train_output = ( + outputs["train_output"].view(1, int(outputs["train_output"].shape[0] ** 0.5), int(outputs["train_output"].shape[0] ** 0.5), 3) + .permute(0, 3, 1, 2) + ) + + with torch.autocast(device_type="cuda", dtype=torch.float16): + sds_loss = self.sd.sds_loss( + text_embedding.to(self.sd_device), + train_output.to(self.sd_device), + guidance_scale=int(self.guidance_scale), + grad_scaler=self.grad_scaler, + ) + + loss_dict["sds_loss"] = sds_loss.to(self.device) + + if self.training: + loss_dict["distortion_loss"] = self.config.distortion_loss_mult * distortion_loss( + outputs["weights_list"], outputs["ray_samples_list"] + ) + loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss( + outputs["weights_list"], outputs["ray_samples_list"] + ) + return loss_dict \ No newline at end of file