mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-19 14:23:57 +08:00
change uniform sampler to prediction network, add option to jit unet for faster sds_loss
This commit is contained in:
1 parent
9e4bc773ad
commit
8ba8942779
4 files changed
+248
-47
No files matched your search
@@ -17,9 +17,11 @@
|
||||
# Modified from https://github.com/ashawkey/stable-dreamfusion/blob/main/nerf/sd.py
|
||||
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple, Union
|
||||
|
||||
import appdirs
|
||||
import mediapy
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -49,6 +51,11 @@ SD_SOURCE = "runwayml/stable-diffusion-v1-5"
|
||||
CLIP_SOURCE = "openai/clip-vit-large-patch14"
|
||||
|
||||
|
||||
@dataclass
|
||||
class UNet2DConditionOutput:
|
||||
sample: torch.FloatTensor
|
||||
|
||||
|
||||
class StableDiffusion(nn.Module):
|
||||
"""Stable Diffusion implementation
|
||||
|
||||
@@ -81,13 +88,34 @@ class StableDiffusion(nn.Module):
|
||||
|
||||
pipe.enable_attention_slicing()
|
||||
|
||||
# use jitted unet
|
||||
unet_traced_filename = Path(appdirs.user_data_dir("nerfstudio")) / "unet_traced.pt"
|
||||
if unet_traced_filename.exists():
|
||||
CONSOLE.print("Loading traced UNet.")
|
||||
unet_traced = torch.jit.load("unet_traced.pt")
|
||||
|
||||
class TracedUNet(torch.nn.Module):
|
||||
"""Jitted version of UNet"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.in_channels = pipe.unet.in_channels
|
||||
self.device = pipe.unet.device
|
||||
|
||||
def forward(self, latent_model_input, t, encoder_hidden_states):
|
||||
sample = unet_traced(latent_model_input, t, encoder_hidden_states)[0]
|
||||
return UNet2DConditionOutput(sample=sample)
|
||||
|
||||
self.unet = TracedUNet()
|
||||
del pipe.unet
|
||||
else:
|
||||
self.unet = pipe.unet
|
||||
self.unet.to(memory_format=torch.channels_last)
|
||||
|
||||
self.tokenizer = pipe.tokenizer
|
||||
self.unet = pipe.unet
|
||||
self.text_encoder = pipe.text_encoder
|
||||
self.auto_encoder = pipe.vae
|
||||
|
||||
self.unet.to(memory_format=torch.channels_last)
|
||||
|
||||
CONSOLE.print("Stable Diffusion loaded!")
|
||||
|
||||
def get_text_embeds(
|
||||
|
||||
@@ -37,13 +37,18 @@ from nerfstudio.engine.callbacks import (
|
||||
|
||||
# from nerfstudio.field_components.encodings import NeRFEncoding,
|
||||
from nerfstudio.field_components.field_heads import FieldHeadNames
|
||||
from nerfstudio.fields.density_fields import HashMLPDensityField
|
||||
from nerfstudio.fields.dreamfusion_field import DreamFusionField
|
||||
from nerfstudio.model_components.losses import (
|
||||
MSELoss,
|
||||
interlevel_loss,
|
||||
orientation_loss,
|
||||
pred_normal_loss,
|
||||
)
|
||||
from nerfstudio.model_components.ray_samplers import UniformSampler
|
||||
from nerfstudio.model_components.ray_samplers import (
|
||||
ProposalNetworkSampler,
|
||||
UniformSampler,
|
||||
)
|
||||
from nerfstudio.model_components.renderers import (
|
||||
AccumulationRenderer,
|
||||
DepthRenderer,
|
||||
@@ -85,6 +90,33 @@ class DreamFusionModelConfig(ModelConfig):
|
||||
background when rendered at the end of training"""
|
||||
transmittance_end_schedule: int = 1000
|
||||
"""number of iterations to reach target_transmittance_end"""
|
||||
num_proposal_samples_per_ray: Tuple[int, ...] = (256, 96)
|
||||
"""Number of samples per ray for each proposal network."""
|
||||
num_nerf_samples_per_ray: int = 48
|
||||
"""Number of samples per ray for the nerf network."""
|
||||
proposal_update_every: int = 5
|
||||
"""Sample every n steps after the warmup"""
|
||||
proposal_warmup: int = 5000
|
||||
"""Scales n from 1 to proposal_update_every over this many steps"""
|
||||
num_proposal_iterations: int = 2
|
||||
"""Number of proposal network iterations."""
|
||||
use_same_proposal_network: bool = False
|
||||
"""Use the same proposal network. Otherwise use different ones."""
|
||||
proposal_net_args_list: List[Dict] = field(
|
||||
default_factory=lambda: [
|
||||
{"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 128},
|
||||
{"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 256},
|
||||
]
|
||||
)
|
||||
"""Arguments for the proposal density fields."""
|
||||
proposal_weights_anneal_slope: float = 10.0
|
||||
"""Slope of the annealing function for the proposal weights."""
|
||||
proposal_weights_anneal_max_num_iters: int = 1000
|
||||
"""Max num iterations for the annealing function."""
|
||||
use_single_jitter: bool = True
|
||||
"""Whether use single jitter or not for the proposal networks."""
|
||||
interlevel_loss_mult: float = 1.0
|
||||
"""Proposal loss multiplier."""
|
||||
|
||||
|
||||
class DreamFusionModel(Model):
|
||||
@@ -109,6 +141,64 @@ class DreamFusionModel(Model):
|
||||
self.target_transmittance = config.target_transmittance_start
|
||||
super().__init__(config=config, **kwargs)
|
||||
|
||||
def populate_modules(self):
|
||||
"""Set the fields and modules"""
|
||||
super().populate_modules()
|
||||
|
||||
# setting up fields
|
||||
self.field = DreamFusionField(self.scene_box.aabb)
|
||||
|
||||
# samplers
|
||||
self.density_fns = []
|
||||
num_prop_nets = self.config.num_proposal_iterations
|
||||
# Build the proposal network(s)
|
||||
self.proposal_networks = torch.nn.ModuleList()
|
||||
|
||||
for i in range(num_prop_nets):
|
||||
prop_net_args = self.config.proposal_net_args_list[min(i, len(self.config.proposal_net_args_list) - 1)]
|
||||
network = HashMLPDensityField(
|
||||
self.scene_box.aabb,
|
||||
**prop_net_args,
|
||||
)
|
||||
self.proposal_networks.append(network)
|
||||
self.density_fns.extend([network.density_fn for network in self.proposal_networks])
|
||||
|
||||
update_schedule = lambda step: np.clip(
|
||||
np.interp(step, [0, self.config.proposal_warmup], [0, self.config.proposal_update_every]),
|
||||
1,
|
||||
self.config.proposal_update_every,
|
||||
)
|
||||
self.proposal_sampler = ProposalNetworkSampler(
|
||||
num_nerf_samples_per_ray=self.config.num_nerf_samples_per_ray,
|
||||
num_proposal_samples_per_ray=self.config.num_proposal_samples_per_ray,
|
||||
num_proposal_network_iterations=self.config.num_proposal_iterations,
|
||||
single_jitter=self.config.use_single_jitter,
|
||||
update_sched=update_schedule,
|
||||
)
|
||||
|
||||
# self.sampler_uniform = UniformSampler(num_samples=self.num_samples, single_jitter=True)
|
||||
|
||||
# renderers
|
||||
self.renderer_rgb = RGBRenderer(background_color=colors.WHITE)
|
||||
self.renderer_accumulation = AccumulationRenderer()
|
||||
self.renderer_depth = DepthRenderer()
|
||||
self.renderer_normals = NormalsRenderer()
|
||||
self.renderer_lambertian = LambertianShadingRenderer()
|
||||
|
||||
# losses
|
||||
self.rgb_loss = MSELoss()
|
||||
|
||||
# metrics
|
||||
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
|
||||
self.ssim = structural_similarity_index_measure
|
||||
self.lpips = LearnedPerceptualImagePatchSimilarity()
|
||||
|
||||
# colliders
|
||||
if self.config.sphere_collider:
|
||||
self.collider = SphereCollider(torch.Tensor([0, 0, 0]), 1.0)
|
||||
else:
|
||||
self.collider = AABBBoxCollider(scene_box=self.scene_box)
|
||||
|
||||
def get_training_callbacks(
|
||||
self, training_callback_attributes: TrainingCallbackAttributes
|
||||
) -> List[TrainingCallback]:
|
||||
@@ -137,6 +227,16 @@ class DreamFusionModel(Model):
|
||||
self.config.target_transmittance_start - self.config.target_transmittance_end
|
||||
)
|
||||
|
||||
# anneal the weights of the proposal network before doing PDF sampling
|
||||
N = self.config.proposal_weights_anneal_max_num_iters
|
||||
|
||||
def set_anneal(step):
|
||||
# https://arxiv.org/pdf/2111.12077.pdf eq. 18
|
||||
train_frac = np.clip(step / N, 0, 1)
|
||||
bias = lambda x, b: (b * x) / ((b - 1) * x + 1)
|
||||
anneal = bias(train_frac, self.config.proposal_weights_anneal_slope)
|
||||
self.proposal_sampler.set_anneal(anneal)
|
||||
|
||||
callbacks = [
|
||||
TrainingCallback(
|
||||
where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION],
|
||||
@@ -162,41 +262,19 @@ class DreamFusionModel(Model):
|
||||
func=update_target_transmittance,
|
||||
args=[self, training_callback_attributes],
|
||||
),
|
||||
TrainingCallback(
|
||||
where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION],
|
||||
update_every_num_iters=1,
|
||||
func=self.proposal_sampler.step_cb,
|
||||
),
|
||||
TrainingCallback(
|
||||
where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION],
|
||||
update_every_num_iters=1,
|
||||
func=set_anneal,
|
||||
),
|
||||
]
|
||||
return callbacks
|
||||
|
||||
def populate_modules(self):
|
||||
"""Set the fields and modules"""
|
||||
super().populate_modules()
|
||||
|
||||
# setting up fields
|
||||
|
||||
self.field = DreamFusionField(self.scene_box.aabb)
|
||||
|
||||
# samplers
|
||||
self.sampler_uniform = UniformSampler(num_samples=self.num_samples, single_jitter=True)
|
||||
|
||||
# renderers
|
||||
self.renderer_rgb = RGBRenderer(background_color=colors.WHITE)
|
||||
self.renderer_accumulation = AccumulationRenderer()
|
||||
self.renderer_depth = DepthRenderer()
|
||||
self.renderer_normals = NormalsRenderer()
|
||||
self.renderer_lambertian = LambertianShadingRenderer()
|
||||
|
||||
# losses
|
||||
self.rgb_loss = MSELoss()
|
||||
|
||||
# metrics
|
||||
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
|
||||
self.ssim = structural_similarity_index_measure
|
||||
self.lpips = LearnedPerceptualImagePatchSimilarity()
|
||||
|
||||
# colliders
|
||||
if self.config.sphere_collider:
|
||||
self.collider = SphereCollider(torch.Tensor([0, 0, 0]), 1.0)
|
||||
else:
|
||||
self.collider = AABBBoxCollider(scene_box=self.scene_box)
|
||||
|
||||
def get_param_groups(self) -> Dict[str, List[Parameter]]:
|
||||
param_groups = {}
|
||||
param_groups["fields"] = list(self.field.parameters())
|
||||
@@ -205,13 +283,16 @@ class DreamFusionModel(Model):
|
||||
def get_outputs(self, ray_bundle: RayBundle):
|
||||
# uniform sampling
|
||||
background_rgb = self.field.get_background_rgb(ray_bundle)
|
||||
# ray_bundle = self.sphere_collider(ray_bundle)
|
||||
ray_samples_uniform = self.sampler_uniform(ray_bundle)
|
||||
field_outputs = self.field(ray_samples_uniform, compute_normals=True)
|
||||
|
||||
ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns)
|
||||
field_outputs = self.field(ray_samples, compute_normals=True)
|
||||
weights = ray_samples.get_weights(field_outputs[FieldHeadNames.DENSITY])
|
||||
weights_list.append(weights)
|
||||
ray_samples_list.append(ray_samples)
|
||||
density = field_outputs[FieldHeadNames.DENSITY]
|
||||
|
||||
if self.initialize_density and self.training:
|
||||
pos = ray_samples_uniform.frustums.get_positions()
|
||||
pos = ray_samples.frustums.get_positions()
|
||||
density_blob = (
|
||||
self.config.init_density_strength
|
||||
* self.density_strength
|
||||
@@ -219,10 +300,8 @@ class DreamFusionModel(Model):
|
||||
)
|
||||
density += density_blob
|
||||
|
||||
weights = ray_samples_uniform.get_weights(density)
|
||||
|
||||
accumulation = self.renderer_accumulation(weights)
|
||||
depth = self.renderer_depth(weights, ray_samples_uniform)
|
||||
depth = self.renderer_depth(weights, ray_samples)
|
||||
rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights)
|
||||
|
||||
accum_mask = torch.clamp((torch.nan_to_num(accumulation, nan=0.0)), min=0.0, max=1.0)
|
||||
@@ -238,6 +317,14 @@ class DreamFusionModel(Model):
|
||||
"depth": depth,
|
||||
}
|
||||
|
||||
# These use a lot of GPU memory, so we avoid storing them for eval.
|
||||
if self.training:
|
||||
outputs["weights_list"] = weights_list
|
||||
outputs["ray_samples_list"] = ray_samples_list
|
||||
|
||||
for i in range(self.config.num_proposal_iterations):
|
||||
outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i])
|
||||
|
||||
normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights)
|
||||
pred_normals = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights) # .detach()
|
||||
|
||||
@@ -313,7 +400,10 @@ class DreamFusionModel(Model):
|
||||
loss_dict["opacity_loss"] = -torch.minimum(
|
||||
(1 - outputs["accumulation"]).mean(), torch.tensor(self.target_transmittance)
|
||||
)
|
||||
|
||||
if self.training:
|
||||
loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss(
|
||||
outputs["weights_list"], outputs["ray_samples_list"]
|
||||
)
|
||||
return loss_dict
|
||||
|
||||
def get_image_metrics_and_images(
|
||||
|
||||
@@ -132,9 +132,9 @@ class DreamfusionPipeline(VanillaPipeline):
|
||||
)
|
||||
loss_dict["sds_loss"] = sds_loss.to(self.device)
|
||||
# TODO: opacity penalty using transmittance, not accumultation
|
||||
if self.config.opacity_penalty:
|
||||
accum_mean = np.mean(1.0 - accumulation)
|
||||
sds_loss *= np.min((0.5, accum_mean))
|
||||
# if self.config.opacity_penalty:
|
||||
# accum_mean = np.mean(1.0 - accumulation)
|
||||
# sds_loss *= np.min((0.5, accum_mean))
|
||||
|
||||
model_outputs["latents"] = latents
|
||||
model_outputs["grad"] = grad
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Trace Stable Diffusion UNet for speed improvement"""
|
||||
|
||||
import functools
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import appdirs
|
||||
import torch
|
||||
from diffusers import StableDiffusionPipeline
|
||||
|
||||
|
||||
def jit_unet():
|
||||
|
||||
# torch disable grad
|
||||
torch.set_grad_enabled(False)
|
||||
|
||||
# set variables
|
||||
n_experiments = 2
|
||||
unet_runs_per_experiment = 50
|
||||
|
||||
# load inputs
|
||||
def generate_inputs():
|
||||
sample = torch.randn(2, 4, 64, 64).half().cuda()
|
||||
timestep = torch.rand(1).half().cuda() * 999
|
||||
encoder_hidden_states = torch.randn(2, 77, 768).half().cuda()
|
||||
return sample, timestep, encoder_hidden_states
|
||||
|
||||
pipe = StableDiffusionPipeline.from_pretrained(
|
||||
"runwayml/stable-diffusion-v1-5",
|
||||
torch_dtype=torch.float16,
|
||||
).to("cuda")
|
||||
|
||||
pipe.enable_attention_slicing()
|
||||
|
||||
unet = pipe.unet
|
||||
unet.eval()
|
||||
unet.to(memory_format=torch.channels_last) # use channels_last memory format
|
||||
unet.forward = functools.partial(unet.forward, return_dict=False) # set return_dict=False as default
|
||||
|
||||
# warmup
|
||||
for _ in range(3):
|
||||
with torch.inference_mode():
|
||||
inputs = generate_inputs()
|
||||
orig_output = unet(*inputs)
|
||||
|
||||
# trace
|
||||
print("tracing..")
|
||||
unet_traced = torch.jit.trace(unet, inputs)
|
||||
unet_traced.eval()
|
||||
print("done tracing")
|
||||
|
||||
# warmup and optimize graph
|
||||
for _ in range(5):
|
||||
with torch.inference_mode():
|
||||
inputs = generate_inputs()
|
||||
orig_output = unet_traced(*inputs)
|
||||
|
||||
# benchmarking
|
||||
with torch.inference_mode():
|
||||
for _ in range(n_experiments):
|
||||
torch.cuda.synchronize()
|
||||
start_time = time.time()
|
||||
for _ in range(unet_runs_per_experiment):
|
||||
orig_output = unet_traced(*inputs)
|
||||
torch.cuda.synchronize()
|
||||
print(f"unet traced inference took {time.time() - start_time:.2f} seconds")
|
||||
for _ in range(n_experiments):
|
||||
torch.cuda.synchronize()
|
||||
start_time = time.time()
|
||||
for _ in range(unet_runs_per_experiment):
|
||||
orig_output = unet(*inputs)
|
||||
torch.cuda.synchronize()
|
||||
print(f"unet inference took {time.time() - start_time:.2f} seconds")
|
||||
|
||||
# save the traced model
|
||||
unet_traced_filename = Path(appdirs.user_data_dir("nerfstudio")) / "unet_traced.pt"
|
||||
if not unet_traced_filename.exists():
|
||||
unet_traced_filename.parent.mkdir(parents=True, exist_ok=True)
|
||||
unet_traced.save(unet_traced_filename)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
jit_unet()
|
||||
Reference in new issue
Block a user