mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-16 21:03:46 +08:00
df plus
This commit is contained in:
1 parent
f7789882b9
commit
0a4d49c4a4
4 files changed
+614
-44
No files matched your search
@@ -30,6 +30,7 @@ from nerfstudio.data.datamanagers.depth_datamanager import DepthDataManagerConfi
|
||||
from nerfstudio.data.datamanagers.dreamfusion_datamanager import (
|
||||
DreamFusionDataManagerConfig,
|
||||
)
|
||||
from nerfstudio.data.datamanagers.dreamfusionplus_datamanager import DreamFusionPlusDataManagerConfig
|
||||
from nerfstudio.data.datamanagers.semantic_datamanager import SemanticDataManagerConfig
|
||||
from nerfstudio.data.datamanagers.variable_res_datamanager import (
|
||||
VariableResDataManagerConfig,
|
||||
@@ -51,6 +52,7 @@ from nerfstudio.engine.trainer import TrainerConfig
|
||||
from nerfstudio.field_components.temporal_distortions import TemporalDistortionKind
|
||||
from nerfstudio.models.depth_nerfacto import DepthNerfactoModelConfig
|
||||
from nerfstudio.models.dreamfusion import DreamFusionModelConfig
|
||||
from nerfstudio.models.dreamfusionplus import DreamFusionPlusModelConfig
|
||||
from nerfstudio.models.instant_ngp import InstantNGPModelConfig
|
||||
from nerfstudio.models.mipnerf import MipNerfModel
|
||||
from nerfstudio.models.nerfacto import NerfactoModelConfig
|
||||
@@ -383,6 +385,53 @@ method_configs["dreamfusion"] = TrainerConfig(
|
||||
vis="viewer",
|
||||
)
|
||||
|
||||
method_configs["dreamplus"] = TrainerConfig(
|
||||
method_name="dreamplus",
|
||||
steps_per_eval_batch=50,
|
||||
steps_per_eval_image=50,
|
||||
steps_per_save=200,
|
||||
max_num_iterations=30000,
|
||||
mixed_precision=True,
|
||||
pipeline=VanillaPipelineConfig(
|
||||
generative=True,
|
||||
datamanager=DreamFusionPlusDataManagerConfig(
|
||||
dataparser=NerfstudioDataParserConfig(),
|
||||
train_num_rays_per_batch=4096,
|
||||
eval_num_rays_per_batch=4096,
|
||||
),
|
||||
model=DreamFusionPlusModelConfig(
|
||||
eval_num_rays_per_chunk=1 << 15,
|
||||
distortion_loss_mult=0.02,
|
||||
interlevel_loss_mult=1.0,
|
||||
orientation_loss_mult=0.1,
|
||||
max_res=256,
|
||||
sphere_collider=True,
|
||||
initialize_density=False,
|
||||
random_background=True,
|
||||
proposal_warmup=1000,
|
||||
proposal_update_every=2,
|
||||
proposal_weights_anneal_max_num_iters=100,
|
||||
start_lambertian_training=1000,
|
||||
start_normals_training=500,
|
||||
opacity_loss_mult=0.001,
|
||||
prompting_type="location_based",
|
||||
guidance_scale=100,
|
||||
),
|
||||
),
|
||||
optimizers={
|
||||
"proposal_networks": {
|
||||
"optimizer": AdamOptimizerConfig(lr=5e-2, eps=1e-15),
|
||||
"scheduler": None,
|
||||
},
|
||||
"fields": {
|
||||
"optimizer": AdamOptimizerConfig(lr=5e-3, eps=1e-15),
|
||||
"scheduler": None,
|
||||
},
|
||||
},
|
||||
viewer=ViewerConfig(num_rays_per_chunk=1 << 15),
|
||||
vis="viewer",
|
||||
)
|
||||
|
||||
method_configs["nerfplayer-nerfacto"] = TrainerConfig(
|
||||
method_name="nerfplayer-nerfacto",
|
||||
pipeline=VanillaPipelineConfig(
|
||||
|
||||
@@ -0,0 +1,338 @@
|
||||
# Copyright 2022 The Nerfstudio Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Data manager for dreamfusion
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple, Type, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
from rich.progress import Console
|
||||
from torch.nn import Parameter
|
||||
from typing_extensions import Literal
|
||||
|
||||
from nerfstudio.cameras.cameras import Cameras
|
||||
from nerfstudio.cameras.rays import RayBundle
|
||||
from nerfstudio.configs.config_utils import to_immutable_dict
|
||||
from nerfstudio.data.datamanagers.base_datamanager import DataManager
|
||||
from nerfstudio.data.datamanagers.dreamfusion_datamanager import (
|
||||
DreamFusionDataManager,
|
||||
DreamFusionDataManagerConfig,
|
||||
random_train_pose,
|
||||
)
|
||||
from nerfstudio.data.datasets.base_dataset import InputDataset
|
||||
from nerfstudio.data.pixel_samplers import PixelSampler
|
||||
from nerfstudio.data.scene_box import SceneBox
|
||||
|
||||
CONSOLE = Console(width=120)
|
||||
|
||||
|
||||
def centralized_jitter(c2w, jitter_std):
|
||||
assert c2w.shape[0] == 1
|
||||
theta = torch.randn(1) * jitter_std
|
||||
phi = torch.randn(1) * jitter_std
|
||||
R = torch.Tensor(
|
||||
[
|
||||
[torch.cos(theta), torch.sin(theta), 0],
|
||||
[-torch.sin(theta), torch.cos(theta), 0],
|
||||
[0, 0, 1],
|
||||
]
|
||||
)
|
||||
R = R @ torch.Tensor(
|
||||
[
|
||||
[1, 0, 0],
|
||||
[0, torch.cos(phi), -torch.sin(phi)],
|
||||
[0, torch.sin(phi), torch.cos(phi)],
|
||||
]
|
||||
)
|
||||
|
||||
R = R.to(c2w.device)
|
||||
|
||||
jittered = copy.deepcopy(c2w)
|
||||
|
||||
jittered[..., :3, :3] = R @ jittered[..., :3, :3]
|
||||
jittered[..., :3, 3] = (R @ jittered[..., :3, 3].unsqueeze(-1)).squeeze(-1)
|
||||
|
||||
return jittered
|
||||
|
||||
|
||||
class TrivialDataset(InputDataset):
|
||||
"""A trivial dataset with blank images for the viewer"""
|
||||
|
||||
# pylint: disable=super-init-not-called
|
||||
def __init__(self, cameras: Cameras, input_img: torch.Tensor):
|
||||
self.size = cameras.size
|
||||
self.cameras = cameras
|
||||
self.alpha_color = None
|
||||
self.scene_box = SceneBox(torch.Tensor([[-1, -1, -1], [1, 1, 1]]))
|
||||
self.mask_filenames = None
|
||||
self.metadata = to_immutable_dict({})
|
||||
self.input_img = input_img
|
||||
|
||||
def __len__(self):
|
||||
return self.size
|
||||
|
||||
def __getitem__(self, index: int) -> Dict:
|
||||
if index == 0:
|
||||
return {"image": self.input_img, "image_idx": index}
|
||||
return {"image": torch.cat([torch.ones(128, 256, 3), torch.zeros(128, 256, 3)], dim=0), "image_idx": index}
|
||||
|
||||
|
||||
@dataclass
|
||||
class DreamFusionPlusDataManagerConfig(DreamFusionDataManagerConfig):
|
||||
"""Configuration for data manager instantiation; DataManager is in charge of keeping the train/eval dataparsers;
|
||||
After instantiation, data manager holds both train/eval datasets and is in charge of returning unpacked
|
||||
train/eval data at each iteration
|
||||
"""
|
||||
|
||||
_target: Type = field(default_factory=lambda: DreamFusionPlusDataManager)
|
||||
input_image_path: Path = Path("data/generative/chair/chair.jpg")
|
||||
"""Directory specifying location of input image."""
|
||||
input_image_mask_path: Optional[Path] = None
|
||||
"""Directory specifying location of input image mask, if one exists"""
|
||||
radius_mean: float = 2.2
|
||||
"""Mean radius of the camera sphere"""
|
||||
use_input_transient: bool = False
|
||||
"""Whether to allow the input image to mask out parts of the image that give poor rgb loss"""
|
||||
input_distance: float = 1.5
|
||||
"""Scalar multiple of radius_mean dictating how close we think this image was taken to the object in question.
|
||||
Large values mean this image was taken relatively far away, small values mean this was taken from relatively
|
||||
close to the object"""
|
||||
input_image_position_jitter_std: float = 0
|
||||
"""Whether or not to jitter the position of the input image each time to prevent accumulation directly
|
||||
in front of the camera. If this is nonozero, it is the standard deviation of the jitter"""
|
||||
reference_directory: Path = Path("")
|
||||
"""Directory specifying location of reference images."""
|
||||
anneal_camera_angles: float = 0
|
||||
"""number of steps to anneal camera angles. We start near the input image and slowly work our way outwards
|
||||
as we train with this greater than 0"""
|
||||
central_rotation_range: Tuple[float, float] = (-180, 180)
|
||||
"""Range of central rotation"""
|
||||
|
||||
|
||||
class DreamFusionPlusDataManager(DreamFusionDataManager): # pylint: disable=abstract-method
|
||||
"""Basic stored data manager implementation.
|
||||
This is pretty much a port over from our old dataloading utilities, and is a little jank
|
||||
under the hood. We may clean this up a little bit under the hood with more standard dataloading
|
||||
components that can be strung together, but it can be just used as a black box for now since
|
||||
only the constructor is likely to change in the future, or maybe passing in step number to the
|
||||
next_train and next_eval functions.
|
||||
Args:
|
||||
config: the DataManagerConfig used to instantiate class
|
||||
"""
|
||||
|
||||
config: DreamFusionPlusDataManagerConfig
|
||||
|
||||
# pylint: disable=super-init-not-called
|
||||
def __init__(
|
||||
self,
|
||||
config: DreamFusionPlusDataManagerConfig,
|
||||
device: Union[torch.device, str] = "cpu",
|
||||
test_mode: Literal["test", "val", "inference"] = "val",
|
||||
world_size: int = 1,
|
||||
local_rank: int = 0,
|
||||
**kwargs, # pylint: disable=unused-argument
|
||||
):
|
||||
self.config = config
|
||||
self.device = device
|
||||
self.world_size = world_size
|
||||
self.local_rank = local_rank
|
||||
self.sampler = None
|
||||
self.test_mode = test_mode
|
||||
self.test_split = "test" if test_mode in ["test", "inference"] else "val"
|
||||
self.dataparser = self.config.dataparser.setup()
|
||||
|
||||
pil_image = Image.open(self.config.input_image_path)
|
||||
image = np.array(pil_image, dtype="uint8") # shape is (h, w) or (h, w, 3 or 4)
|
||||
if len(image.shape) == 2:
|
||||
image = image[:, :, None].repeat(3, axis=2)
|
||||
assert image.shape[-1] == 3, "Image must be RGB"
|
||||
image = torch.from_numpy(image).to(self.device).float() / 255.0
|
||||
self.input_image = image.to(device)
|
||||
self.input_image_mask = None
|
||||
if self.config.input_image_mask_path is not None:
|
||||
pil_mask = Image.open(self.config.input_image_mask_path)
|
||||
mask = torch.from_numpy(np.array(pil_mask))
|
||||
self.input_image_mask = torch.any(mask != 0, dim=-1)
|
||||
|
||||
# Transients for the input image, allows it to mask out parts of the image that are background
|
||||
|
||||
original_cameras, _, _ = random_train_pose(
|
||||
1,
|
||||
1,
|
||||
device=torch.device("cpu"),
|
||||
radius_mean=self.config.radius_mean * self.config.input_distance,
|
||||
radius_std=0,
|
||||
focal_range=[1, 1],
|
||||
vertical_rotation_range=[0, 0],
|
||||
central_rotation_range=[0, 0],
|
||||
jitter_std=0,
|
||||
)
|
||||
|
||||
original_cameras = Cameras(
|
||||
camera_to_worlds=original_cameras.camera_to_worlds,
|
||||
fx=float(image.shape[1]),
|
||||
fy=float(image.shape[0]),
|
||||
cx=image.shape[1] / 2,
|
||||
cy=image.shape[0] / 2,
|
||||
)
|
||||
self.original_cameras = original_cameras.to(self.device)
|
||||
|
||||
cameras, _, _ = random_train_pose(
|
||||
self.config.num_eval_angles,
|
||||
self.config.eval_resolution,
|
||||
device=self.device,
|
||||
radius_mean=self.config.radius_mean,
|
||||
radius_std=self.config.radius_std,
|
||||
focal_range=self.config.focal_range,
|
||||
vertical_rotation_range=self.config.vertical_rotation_range,
|
||||
central_rotation_range=self.config.central_rotation_range,
|
||||
jitter_std=self.config.jitter_std,
|
||||
)
|
||||
cameras = cameras.to("cpu")
|
||||
|
||||
cameras = Cameras(
|
||||
camera_to_worlds=torch.cat([original_cameras.camera_to_worlds, cameras.camera_to_worlds], dim=0),
|
||||
fx=torch.cat([original_cameras.fx, cameras.fx], dim=0),
|
||||
fy=torch.cat([original_cameras.fy, cameras.fy], dim=0),
|
||||
cx=torch.cat([original_cameras.cx, cameras.cx], dim=0),
|
||||
cy=torch.cat([original_cameras.cy, cameras.cy], dim=0),
|
||||
width=torch.cat([original_cameras.width, cameras.width], dim=0),
|
||||
height=torch.cat([original_cameras.height, cameras.height], dim=0),
|
||||
)
|
||||
|
||||
self.train_dataset = TrivialDataset(cameras, image)
|
||||
self.eval_dataset = TrivialDataset(cameras, image)
|
||||
|
||||
self.pixel_sampler = PixelSampler(self.config.train_resolution**2)
|
||||
|
||||
# pylint: disable=non-parent-init-called
|
||||
DataManager.__init__(self)
|
||||
if self.config.use_input_transient:
|
||||
self.input_mask = Parameter(torch.ones(image.shape[:2]).float(), requires_grad=True)
|
||||
|
||||
def next_input(self, step: int) -> Tuple[RayBundle, Dict]:
|
||||
"""Returns the batch for the input image"""
|
||||
|
||||
if self.input_image_mask is None:
|
||||
batch = self.pixel_sampler.sample({"image": self.input_image.unsqueeze(0), "image_idx": torch.tensor([0])})
|
||||
else:
|
||||
batch = self.pixel_sampler.sample(
|
||||
{"image": self.input_image.unsqueeze(0), "image_idx": torch.tensor([0]), "mask": self.input_image_mask}
|
||||
)
|
||||
|
||||
batch["input_image"] = True
|
||||
|
||||
ray_indices = batch["indices"]
|
||||
c = ray_indices[:, 0] # camera indices
|
||||
y = ray_indices[:, 1] # row indices
|
||||
x = ray_indices[:, 2] # col indices
|
||||
coords = self.original_cameras.get_image_coords()[y, x]
|
||||
if self.config.use_input_transient:
|
||||
input_mask_pixels = torch.sigmoid(self.input_mask[y, x])
|
||||
batch["input_mask"] = input_mask_pixels
|
||||
|
||||
original_camera = self.original_cameras
|
||||
if self.config.jitter_std:
|
||||
original_camera = copy.deepcopy(original_camera)
|
||||
original_camera.camera_to_worlds = centralized_jitter(
|
||||
original_camera.camera_to_worlds, self.config.input_image_position_jitter_std * min(step, 300) / 300
|
||||
)
|
||||
ray_bundle = original_camera.generate_rays(
|
||||
camera_indices=c.unsqueeze(-1),
|
||||
coords=coords,
|
||||
)
|
||||
return ray_bundle, batch
|
||||
|
||||
def next_train(self, step: int) -> Tuple[RayBundle, Dict]:
|
||||
"""Returns the next batch of data from the train dataloader."""
|
||||
|
||||
self.train_count += 1
|
||||
|
||||
# randomly choose a random camera view or the input view here
|
||||
if np.random.random_sample() < 0.1:
|
||||
return self.next_input(step)
|
||||
|
||||
# # TODO Reimplement when cameras are fully working
|
||||
# if step > 2000:
|
||||
# cameras, _, _ = random_train_pose(
|
||||
# self.config.train_images_per_batch, self.config.train_resolution, device=self.device
|
||||
# )
|
||||
|
||||
# ray_bundle = cameras.generate_rays(
|
||||
# torch.tensor(list(range(self.config.train_images_per_batch)))
|
||||
# ).flatten()
|
||||
# return ray_bundle, {"initialization": False}
|
||||
|
||||
# TODO below
|
||||
if step < self.config.anneal_camera_angles and self.config.anneal_camera_angles > 0:
|
||||
cameras, vertical_rotation, central_rotation = random_train_pose(
|
||||
self.config.train_images_per_batch,
|
||||
self.config.train_resolution,
|
||||
device=self.device,
|
||||
radius_mean=self.config.radius_mean,
|
||||
radius_std=self.config.radius_std,
|
||||
focal_range=self.config.focal_range,
|
||||
vertical_rotation_range=torch.tensor(self.config.vertical_rotation_range)
|
||||
* step
|
||||
/ (self.config.anneal_camera_angles),
|
||||
central_rotation_range=torch.tensor(self.config.central_rotation_range)
|
||||
* step
|
||||
/ (self.config.anneal_camera_angles),
|
||||
jitter_std=self.config.jitter_std,
|
||||
)
|
||||
else:
|
||||
cameras, vertical_rotation, central_rotation = random_train_pose(
|
||||
self.config.train_images_per_batch,
|
||||
self.config.train_resolution,
|
||||
device=self.device,
|
||||
radius_mean=self.config.radius_mean,
|
||||
radius_std=self.config.radius_std,
|
||||
focal_range=self.config.focal_range,
|
||||
vertical_rotation_range=self.config.vertical_rotation_range,
|
||||
central_rotation_range=self.config.central_rotation_range,
|
||||
jitter_std=self.config.jitter_std,
|
||||
)
|
||||
ray_bundle = cameras.generate_rays(torch.tensor(list(range(self.config.train_images_per_batch)))).flatten()
|
||||
|
||||
# camera_idx = torch.randint(0, self.eval_cameras.shape[0], [1], dtype=torch.long, device=self.device)
|
||||
# ray_bundle = self.eval_cameras.generate_rays(camera_idx).flatten()
|
||||
|
||||
return ray_bundle, {"vertical": vertical_rotation, "central": central_rotation, "initialization": True, "input_image": False}
|
||||
|
||||
def get_param_groups(self) -> Dict[str, List[Parameter]]: # pylint: disable=no-self-use
|
||||
"""Get the param groups for the data manager.
|
||||
Returns:
|
||||
A list of dictionaries containing the data manager's param groups.
|
||||
"""
|
||||
if self.config.use_input_transient:
|
||||
param_groups = {"transients": [self.input_mask]}
|
||||
else:
|
||||
return {}
|
||||
|
||||
return param_groups
|
||||
|
||||
def setup_train(self):
|
||||
pass
|
||||
|
||||
def setup_eval(self):
|
||||
pass
|
||||
@@ -12,18 +12,6 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
<<<<<<< HEAD
|
||||
"""Utility functions for Stable Diffusion"""
|
||||
|
||||
def get_text_embedding(batch, prompting_type, prompts, device):
|
||||
"""util function for getting correct text_embedding from view direction
|
||||
Args:
|
||||
batch: contains ray batch information
|
||||
prompting_type: how to sample text embeddings
|
||||
prompts: dictionary of text embeddings
|
||||
|
||||
"""
|
||||
=======
|
||||
"""
|
||||
Stable diffusion utils
|
||||
"""
|
||||
@@ -37,41 +25,10 @@ def get_text_embedding(batch, prompting_type, text_embeddings, sd_device):
|
||||
top = text_embeddings["top_text_embedding"]
|
||||
back = text_embeddings["back_text_embedding"]
|
||||
|
||||
>>>>>>> a474aa3fb81d07d55beb27685b4e8d6cd5bf9514
|
||||
assert prompting_type in ("location_based", "interpolated", "base")
|
||||
|
||||
if prompting_type == "location_based":
|
||||
if batch["vertical"] < 40:
|
||||
<<<<<<< HEAD
|
||||
text_embedding = prompts["top_text_embedding"]
|
||||
elif batch["central"] > 315 or batch["central"] <= 45:
|
||||
text_embedding = prompts["front_text_embedding"]
|
||||
elif batch["central"] > 45 and batch["central"] <= 135:
|
||||
text_embedding = prompts["side_text_embedding"]
|
||||
elif batch["central"] > 135 and batch["central"] <= 225:
|
||||
text_embedding = prompts["back_text_embedding"]
|
||||
else: # batch["central"] > 225 and batch["central"] <= 315:
|
||||
text_embedding = prompts["side_text_embedding"]
|
||||
|
||||
elif prompting_type == "interpolated":
|
||||
horiz = batch["central"].to(device)
|
||||
vert = max(batch["vertical"].to(device), 0)
|
||||
|
||||
if batch["central"] > 0 or batch["central"] <= 90:
|
||||
text_embedding = (horiz) * prompts["side_text_embedding"] + (90 - horiz) * prompts["front_text_embedding"]
|
||||
elif batch["central"] > 90 and batch["central"] <= 180:
|
||||
text_embedding = (horiz - 90) * prompts["back_text_embedding"] + (180 - horiz) * prompts["side_text_embedding"]
|
||||
elif batch["central"] > 180 and batch["central"] <= 270:
|
||||
text_embedding = (horiz - 180) * prompts["side_text_embedding"] + (270 - horiz) * prompts["back_text_embedding"]
|
||||
else: # batch["central"] > 270 and batch["central"] <= 360:
|
||||
text_embedding = (horiz - 270) * prompts["front_text_embedding"] + (360 - horiz) * prompts["side_text_embedding"]
|
||||
|
||||
text_embedding = text_embedding / 90.0
|
||||
text_embedding = (vert * text_embedding + (90 - vert) * prompts["top_text_embedding"]) / 90.0
|
||||
|
||||
else:
|
||||
text_embedding = prompts["base_text_embedding"]
|
||||
=======
|
||||
text_embedding = top
|
||||
elif batch["central"] > 315 or batch["central"] <= 45:
|
||||
text_embedding = front
|
||||
@@ -100,6 +57,5 @@ def get_text_embedding(batch, prompting_type, text_embeddings, sd_device):
|
||||
|
||||
elif prompting_type == "base":
|
||||
text_embedding = base
|
||||
>>>>>>> a474aa3fb81d07d55beb27685b4e8d6cd5bf9514
|
||||
|
||||
return text_embedding
|
||||
@@ -0,0 +1,227 @@
|
||||
# Copyright 2022 The Nerfstudio Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
DreamFusion implementation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Tuple, Type, Optional
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch.nn import Parameter
|
||||
from torch.cuda.amp.grad_scaler import GradScaler
|
||||
|
||||
from nerfstudio.cameras.rays import RayBundle
|
||||
from nerfstudio.engine.callbacks import (
|
||||
TrainingCallback,
|
||||
TrainingCallbackAttributes,
|
||||
TrainingCallbackLocation,
|
||||
)
|
||||
from nerfstudio.generative.stable_diffusion import StableDiffusion
|
||||
from nerfstudio.generative.stable_diffusion_utils import get_text_embedding
|
||||
from nerfstudio.field_components.field_heads import FieldHeadNames
|
||||
from nerfstudio.fields.density_fields import HashMLPDensityField
|
||||
from nerfstudio.fields.dreamfusion_field import DreamFusionField
|
||||
from nerfstudio.model_components.losses import (
|
||||
MSELoss,
|
||||
distortion_loss,
|
||||
interlevel_loss,
|
||||
orientation_loss,
|
||||
pred_normal_loss,
|
||||
)
|
||||
from nerfstudio.model_components.ray_samplers import (
|
||||
ProposalNetworkSampler,
|
||||
UniformSampler,
|
||||
)
|
||||
from nerfstudio.model_components.renderers import (
|
||||
AccumulationRenderer,
|
||||
DepthRenderer,
|
||||
NormalsRenderer,
|
||||
RGBRenderer,
|
||||
)
|
||||
from nerfstudio.model_components.scene_colliders import AABBBoxCollider, SphereCollider
|
||||
from nerfstudio.model_components.shaders import LambertianShader, NormalsShader
|
||||
from nerfstudio.models.base_model import Model, ModelConfig
|
||||
from nerfstudio.models.dreamfusion import DreamFusionModel, DreamFusionModelConfig
|
||||
from nerfstudio.utils import colormaps, colors, math, misc
|
||||
|
||||
|
||||
@dataclass
|
||||
class DreamFusionPlusModelConfig(DreamFusionModelConfig):
|
||||
"""DreamFusion model config"""
|
||||
|
||||
_target: Type = field(default_factory=lambda: DreamFusionPlusModel)
|
||||
"""target class to instantiate"""
|
||||
prompt: str = "A high-quality photo of a tree frog on a stump"
|
||||
"""prompt for stable dreamfusion"""
|
||||
|
||||
orientation_loss_mult: float = 0.0001
|
||||
"""Orientation loss multipier on computed normals."""
|
||||
pred_normal_loss_mult: float = 0.001
|
||||
"""Predicted normal loss multiplier."""
|
||||
random_light_source: bool = True
|
||||
"""Randomizes light source per output."""
|
||||
initialize_density: bool = True
|
||||
"""Initialize density in center of scene."""
|
||||
init_density_strength: float = 0.1
|
||||
"""Initial strength of center density"""
|
||||
sphere_collider: bool = True
|
||||
"""Use spherical collider instead of box"""
|
||||
random_background: bool = True
|
||||
"""Randomly choose between using background mlp and random color for background"""
|
||||
target_transmittance_start: float = 0.4
|
||||
"""target transmittance for opacity penalty. This is the percent of the scene that is
|
||||
background when rendered at the start of training"""
|
||||
target_transmittance_end: float = 0.7
|
||||
"""target transmittance for opacity penalty. This is the percent of the scene that is
|
||||
background when rendered at the end of training"""
|
||||
transmittance_end_schedule: int = 1500
|
||||
"""number of iterations to reach target_transmittance_end"""
|
||||
num_proposal_samples_per_ray: Tuple[int, ...] = (256, 96)
|
||||
"""Number of samples per ray for each proposal network."""
|
||||
num_nerf_samples_per_ray: int = 48
|
||||
"""Number of samples per ray for the nerf network."""
|
||||
proposal_update_every: int = 5
|
||||
"""Sample every n steps after the warmup"""
|
||||
proposal_warmup: int = 5000
|
||||
"""Scales n from 1 to proposal_update_every over this many steps"""
|
||||
num_proposal_iterations: int = 2
|
||||
"""Number of proposal network iterations."""
|
||||
use_same_proposal_network: bool = False
|
||||
"""Use the same proposal network. Otherwise use different ones."""
|
||||
proposal_net_args_list: List[Dict] = field(
|
||||
default_factory=lambda: [
|
||||
{"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 128},
|
||||
{"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 256},
|
||||
]
|
||||
)
|
||||
"""Arguments for the proposal density fields."""
|
||||
proposal_weights_anneal_slope: float = 10.0
|
||||
"""Slope of the annealing function for the proposal weights."""
|
||||
proposal_weights_anneal_max_num_iters: int = 500
|
||||
"""Max num iterations for the annealing function."""
|
||||
use_single_jitter: bool = True
|
||||
"""Whether use single jitter or not for the proposal networks."""
|
||||
interlevel_loss_mult: float = 1.0
|
||||
"""Proposal loss multiplier."""
|
||||
distortion_loss_mult: float = 1.0
|
||||
"""Distortion loss multiplier."""
|
||||
start_normals_training: int = 1000
|
||||
"""Start training normals after this many iterations"""
|
||||
start_lambertian_training: int = 1000
|
||||
"""start training with lambertian shading after this many iterations"""
|
||||
opacity_penalty: bool = True
|
||||
"""enables penalty to encourage sparse weights (penalizing for uniform density along ray)"""
|
||||
opacity_loss_mult: float = 1
|
||||
"""scale for opacity penalty"""
|
||||
max_res: int = 256
|
||||
"""Maximum resolution of the density field."""
|
||||
|
||||
prompting_type: str = "location_based"
|
||||
"""choose between location_based, interpolated, base"""
|
||||
top_prompt: str = ", overhead view"
|
||||
"""appended to prompt for overhead view"""
|
||||
side_prompt: str = ", side view"
|
||||
"""appended to prompt for side view"""
|
||||
front_prompt: str = ", front view"
|
||||
"""appended to prompt for front view"""
|
||||
back_prompt: str = ", back view"
|
||||
"""appended to prompt for back view"""
|
||||
guidance_scale: float = 100
|
||||
"""guidance scale for sds loss"""
|
||||
stablediffusion_device: Optional[str] = None
|
||||
"""device for stable diffusion"""
|
||||
sd_version: str = '1-5'
|
||||
|
||||
|
||||
class DreamFusionPlusModel(DreamFusionModel):
|
||||
"""DreamFusionModel Model
|
||||
|
||||
Args:
|
||||
config: DreamFusion configuration to instantiate model
|
||||
"""
|
||||
|
||||
config: DreamFusionPlusModelConfig
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: DreamFusionPlusModelConfig,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(config=config, **kwargs)
|
||||
|
||||
|
||||
def get_loss_dict(self, outputs, batch, metrics_dict=None) -> Dict[str, torch.Tensor]:
|
||||
# Scaling metrics by coefficients to create the losses.
|
||||
|
||||
loss_dict = {}
|
||||
loss_dict = misc.scale_dict(loss_dict, self.config.loss_coefficients)
|
||||
if self.train_normals:
|
||||
# orientation loss for computed normals
|
||||
loss_dict["orientation_loss"] = self.config.orientation_loss_mult * torch.mean(
|
||||
outputs["rendered_orientation_loss"]
|
||||
)
|
||||
# ground truth supervision for normals
|
||||
loss_dict["pred_normal_loss"] = self.config.pred_normal_loss_mult * torch.mean(
|
||||
outputs["rendered_pred_normal_loss"]
|
||||
)
|
||||
else:
|
||||
loss_dict["orientation_loss"] = 0
|
||||
loss_dict["pred_normal_loss"] = 0
|
||||
|
||||
if self.config.opacity_penalty:
|
||||
loss_dict["opacity_loss"] = self.config.opacity_loss_mult * outputs["opacity_loss"].mean()
|
||||
|
||||
if self.prompt != self.cur_prompt:
|
||||
self.cur_prompt = self.prompt
|
||||
self.text_embeddings = {
|
||||
"top_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.top_prompt}", ""),
|
||||
"front_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.front_prompt}", ""),
|
||||
"side_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.side_prompt}", ""),
|
||||
"back_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.back_prompt}", ""),
|
||||
"base_text_embedding": self.sd.get_text_embeds(self.cur_prompt, "")
|
||||
}
|
||||
|
||||
if batch["input_image"]:
|
||||
loss_dict["rgb_loss"] = self.rgb_loss(batch["image"].to(self.device), outputs["rgb"])
|
||||
else:
|
||||
text_embedding = get_text_embedding(batch, self.prompting_type, self.text_embeddings, self.sd_device)
|
||||
|
||||
train_output = (
|
||||
outputs["train_output"].view(1, int(outputs["train_output"].shape[0] ** 0.5), int(outputs["train_output"].shape[0] ** 0.5), 3)
|
||||
.permute(0, 3, 1, 2)
|
||||
)
|
||||
|
||||
with torch.autocast(device_type="cuda", dtype=torch.float16):
|
||||
sds_loss = self.sd.sds_loss(
|
||||
text_embedding.to(self.sd_device),
|
||||
train_output.to(self.sd_device),
|
||||
guidance_scale=int(self.guidance_scale),
|
||||
grad_scaler=self.grad_scaler,
|
||||
)
|
||||
|
||||
loss_dict["sds_loss"] = sds_loss.to(self.device)
|
||||
|
||||
if self.training:
|
||||
loss_dict["distortion_loss"] = self.config.distortion_loss_mult * distortion_loss(
|
||||
outputs["weights_list"], outputs["ray_samples_list"]
|
||||
)
|
||||
loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss(
|
||||
outputs["weights_list"], outputs["ray_samples_list"]
|
||||
)
|
||||
return loss_dict
|
||||
Reference in new issue
Block a user