change uniform sampler to prediction network, add option to jit unet for faster sds_loss

This commit is contained in:
terrance committed 2023-02-03 01:42:05 +00:00
1 parent 9e4bc773ad
commit 8ba8942779
4 files changed
+248 -47

No files matched your search

+31 -3
View File
@@ -17,9 +17,11 @@
# Modified from https://github.com/ashawkey/stable-dreamfusion/blob/main/nerf/sd.py
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import List, Optional, Tuple, Union
import appdirs
import mediapy
import numpy as np
import torch
@@ -49,6 +51,11 @@ SD_SOURCE = "runwayml/stable-diffusion-v1-5"
CLIP_SOURCE = "openai/clip-vit-large-patch14"
@dataclass
class UNet2DConditionOutput:
sample: torch.FloatTensor
class StableDiffusion(nn.Module):
"""Stable Diffusion implementation
@@ -81,13 +88,34 @@ class StableDiffusion(nn.Module):
pipe.enable_attention_slicing()
# use jitted unet
unet_traced_filename = Path(appdirs.user_data_dir("nerfstudio")) / "unet_traced.pt"
if unet_traced_filename.exists():
CONSOLE.print("Loading traced UNet.")
unet_traced = torch.jit.load("unet_traced.pt")
class TracedUNet(torch.nn.Module):
"""Jitted version of UNet"""
def __init__(self):
super().__init__()
self.in_channels = pipe.unet.in_channels
self.device = pipe.unet.device
def forward(self, latent_model_input, t, encoder_hidden_states):
sample = unet_traced(latent_model_input, t, encoder_hidden_states)[0]
return UNet2DConditionOutput(sample=sample)
self.unet = TracedUNet()
del pipe.unet
else:
self.unet = pipe.unet
self.unet.to(memory_format=torch.channels_last)
self.tokenizer = pipe.tokenizer
self.unet = pipe.unet
self.text_encoder = pipe.text_encoder
self.auto_encoder = pipe.vae
self.unet.to(memory_format=torch.channels_last)
CONSOLE.print("Stable Diffusion loaded!")
def get_text_embeds(
+131 -41
View File
@@ -37,13 +37,18 @@ from nerfstudio.engine.callbacks import (
# from nerfstudio.field_components.encodings import NeRFEncoding,
from nerfstudio.field_components.field_heads import FieldHeadNames
from nerfstudio.fields.density_fields import HashMLPDensityField
from nerfstudio.fields.dreamfusion_field import DreamFusionField
from nerfstudio.model_components.losses import (
MSELoss,
interlevel_loss,
orientation_loss,
pred_normal_loss,
)
from nerfstudio.model_components.ray_samplers import UniformSampler
from nerfstudio.model_components.ray_samplers import (
ProposalNetworkSampler,
UniformSampler,
)
from nerfstudio.model_components.renderers import (
AccumulationRenderer,
DepthRenderer,
@@ -85,6 +90,33 @@ class DreamFusionModelConfig(ModelConfig):
background when rendered at the end of training"""
transmittance_end_schedule: int = 1000
"""number of iterations to reach target_transmittance_end"""
num_proposal_samples_per_ray: Tuple[int, ...] = (256, 96)
"""Number of samples per ray for each proposal network."""
num_nerf_samples_per_ray: int = 48
"""Number of samples per ray for the nerf network."""
proposal_update_every: int = 5
"""Sample every n steps after the warmup"""
proposal_warmup: int = 5000
"""Scales n from 1 to proposal_update_every over this many steps"""
num_proposal_iterations: int = 2
"""Number of proposal network iterations."""
use_same_proposal_network: bool = False
"""Use the same proposal network. Otherwise use different ones."""
proposal_net_args_list: List[Dict] = field(
default_factory=lambda: [
{"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 128},
{"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 256},
]
)
"""Arguments for the proposal density fields."""
proposal_weights_anneal_slope: float = 10.0
"""Slope of the annealing function for the proposal weights."""
proposal_weights_anneal_max_num_iters: int = 1000
"""Max num iterations for the annealing function."""
use_single_jitter: bool = True
"""Whether use single jitter or not for the proposal networks."""
interlevel_loss_mult: float = 1.0
"""Proposal loss multiplier."""
class DreamFusionModel(Model):
@@ -109,6 +141,64 @@ class DreamFusionModel(Model):
self.target_transmittance = config.target_transmittance_start
super().__init__(config=config, **kwargs)
def populate_modules(self):
"""Set the fields and modules"""
super().populate_modules()
# setting up fields
self.field = DreamFusionField(self.scene_box.aabb)
# samplers
self.density_fns = []
num_prop_nets = self.config.num_proposal_iterations
# Build the proposal network(s)
self.proposal_networks = torch.nn.ModuleList()
for i in range(num_prop_nets):
prop_net_args = self.config.proposal_net_args_list[min(i, len(self.config.proposal_net_args_list) - 1)]
network = HashMLPDensityField(
self.scene_box.aabb,
**prop_net_args,
)
self.proposal_networks.append(network)
self.density_fns.extend([network.density_fn for network in self.proposal_networks])
update_schedule = lambda step: np.clip(
np.interp(step, [0, self.config.proposal_warmup], [0, self.config.proposal_update_every]),
1,
self.config.proposal_update_every,
)
self.proposal_sampler = ProposalNetworkSampler(
num_nerf_samples_per_ray=self.config.num_nerf_samples_per_ray,
num_proposal_samples_per_ray=self.config.num_proposal_samples_per_ray,
num_proposal_network_iterations=self.config.num_proposal_iterations,
single_jitter=self.config.use_single_jitter,
update_sched=update_schedule,
)
# self.sampler_uniform = UniformSampler(num_samples=self.num_samples, single_jitter=True)
# renderers
self.renderer_rgb = RGBRenderer(background_color=colors.WHITE)
self.renderer_accumulation = AccumulationRenderer()
self.renderer_depth = DepthRenderer()
self.renderer_normals = NormalsRenderer()
self.renderer_lambertian = LambertianShadingRenderer()
# losses
self.rgb_loss = MSELoss()
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
self.lpips = LearnedPerceptualImagePatchSimilarity()
# colliders
if self.config.sphere_collider:
self.collider = SphereCollider(torch.Tensor([0, 0, 0]), 1.0)
else:
self.collider = AABBBoxCollider(scene_box=self.scene_box)
def get_training_callbacks(
self, training_callback_attributes: TrainingCallbackAttributes
) -> List[TrainingCallback]:
@@ -137,6 +227,16 @@ class DreamFusionModel(Model):
self.config.target_transmittance_start - self.config.target_transmittance_end
)
# anneal the weights of the proposal network before doing PDF sampling
N = self.config.proposal_weights_anneal_max_num_iters
def set_anneal(step):
# https://arxiv.org/pdf/2111.12077.pdf eq. 18
train_frac = np.clip(step / N, 0, 1)
bias = lambda x, b: (b * x) / ((b - 1) * x + 1)
anneal = bias(train_frac, self.config.proposal_weights_anneal_slope)
self.proposal_sampler.set_anneal(anneal)
callbacks = [
TrainingCallback(
where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION],
@@ -162,41 +262,19 @@ class DreamFusionModel(Model):
func=update_target_transmittance,
args=[self, training_callback_attributes],
),
TrainingCallback(
where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION],
update_every_num_iters=1,
func=self.proposal_sampler.step_cb,
),
TrainingCallback(
where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION],
update_every_num_iters=1,
func=set_anneal,
),
]
return callbacks
def populate_modules(self):
"""Set the fields and modules"""
super().populate_modules()
# setting up fields
self.field = DreamFusionField(self.scene_box.aabb)
# samplers
self.sampler_uniform = UniformSampler(num_samples=self.num_samples, single_jitter=True)
# renderers
self.renderer_rgb = RGBRenderer(background_color=colors.WHITE)
self.renderer_accumulation = AccumulationRenderer()
self.renderer_depth = DepthRenderer()
self.renderer_normals = NormalsRenderer()
self.renderer_lambertian = LambertianShadingRenderer()
# losses
self.rgb_loss = MSELoss()
# metrics
self.psnr = PeakSignalNoiseRatio(data_range=1.0)
self.ssim = structural_similarity_index_measure
self.lpips = LearnedPerceptualImagePatchSimilarity()
# colliders
if self.config.sphere_collider:
self.collider = SphereCollider(torch.Tensor([0, 0, 0]), 1.0)
else:
self.collider = AABBBoxCollider(scene_box=self.scene_box)
def get_param_groups(self) -> Dict[str, List[Parameter]]:
param_groups = {}
param_groups["fields"] = list(self.field.parameters())
@@ -205,13 +283,16 @@ class DreamFusionModel(Model):
def get_outputs(self, ray_bundle: RayBundle):
# uniform sampling
background_rgb = self.field.get_background_rgb(ray_bundle)
# ray_bundle = self.sphere_collider(ray_bundle)
ray_samples_uniform = self.sampler_uniform(ray_bundle)
field_outputs = self.field(ray_samples_uniform, compute_normals=True)
ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns)
field_outputs = self.field(ray_samples, compute_normals=True)
weights = ray_samples.get_weights(field_outputs[FieldHeadNames.DENSITY])
weights_list.append(weights)
ray_samples_list.append(ray_samples)
density = field_outputs[FieldHeadNames.DENSITY]
if self.initialize_density and self.training:
pos = ray_samples_uniform.frustums.get_positions()
pos = ray_samples.frustums.get_positions()
density_blob = (
self.config.init_density_strength
* self.density_strength
@@ -219,10 +300,8 @@ class DreamFusionModel(Model):
)
density += density_blob
weights = ray_samples_uniform.get_weights(density)
accumulation = self.renderer_accumulation(weights)
depth = self.renderer_depth(weights, ray_samples_uniform)
depth = self.renderer_depth(weights, ray_samples)
rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights)
accum_mask = torch.clamp((torch.nan_to_num(accumulation, nan=0.0)), min=0.0, max=1.0)
@@ -238,6 +317,14 @@ class DreamFusionModel(Model):
"depth": depth,
}
# These use a lot of GPU memory, so we avoid storing them for eval.
if self.training:
outputs["weights_list"] = weights_list
outputs["ray_samples_list"] = ray_samples_list
for i in range(self.config.num_proposal_iterations):
outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i])
normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights)
pred_normals = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights) # .detach()
@@ -313,7 +400,10 @@ class DreamFusionModel(Model):
loss_dict["opacity_loss"] = -torch.minimum(
(1 - outputs["accumulation"]).mean(), torch.tensor(self.target_transmittance)
)
if self.training:
loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss(
outputs["weights_list"], outputs["ray_samples_list"]
)
return loss_dict
def get_image_metrics_and_images(
+3 -3
View File
@@ -132,9 +132,9 @@ class DreamfusionPipeline(VanillaPipeline):
)
loss_dict["sds_loss"] = sds_loss.to(self.device)
# TODO: opacity penalty using transmittance, not accumultation
if self.config.opacity_penalty:
accum_mean = np.mean(1.0 - accumulation)
sds_loss *= np.min((0.5, accum_mean))
# if self.config.opacity_penalty:
# accum_mean = np.mean(1.0 - accumulation)
# sds_loss *= np.min((0.5, accum_mean))
model_outputs["latents"] = latents
model_outputs["grad"] = grad
+83
View File
@@ -0,0 +1,83 @@
"""Trace Stable Diffusion UNet for speed improvement"""
import functools
import time
from pathlib import Path
import appdirs
import torch
from diffusers import StableDiffusionPipeline
def jit_unet():
# torch disable grad
torch.set_grad_enabled(False)
# set variables
n_experiments = 2
unet_runs_per_experiment = 50
# load inputs
def generate_inputs():
sample = torch.randn(2, 4, 64, 64).half().cuda()
timestep = torch.rand(1).half().cuda() * 999
encoder_hidden_states = torch.randn(2, 77, 768).half().cuda()
return sample, timestep, encoder_hidden_states
pipe = StableDiffusionPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5",
torch_dtype=torch.float16,
).to("cuda")
pipe.enable_attention_slicing()
unet = pipe.unet
unet.eval()
unet.to(memory_format=torch.channels_last) # use channels_last memory format
unet.forward = functools.partial(unet.forward, return_dict=False) # set return_dict=False as default
# warmup
for _ in range(3):
with torch.inference_mode():
inputs = generate_inputs()
orig_output = unet(*inputs)
# trace
print("tracing..")
unet_traced = torch.jit.trace(unet, inputs)
unet_traced.eval()
print("done tracing")
# warmup and optimize graph
for _ in range(5):
with torch.inference_mode():
inputs = generate_inputs()
orig_output = unet_traced(*inputs)
# benchmarking
with torch.inference_mode():
for _ in range(n_experiments):
torch.cuda.synchronize()
start_time = time.time()
for _ in range(unet_runs_per_experiment):
orig_output = unet_traced(*inputs)
torch.cuda.synchronize()
print(f"unet traced inference took {time.time() - start_time:.2f} seconds")
for _ in range(n_experiments):
torch.cuda.synchronize()
start_time = time.time()
for _ in range(unet_runs_per_experiment):
orig_output = unet(*inputs)
torch.cuda.synchronize()
print(f"unet inference took {time.time() - start_time:.2f} seconds")
# save the traced model
unet_traced_filename = Path(appdirs.user_data_dir("nerfstudio")) / "unet_traced.pt"
if not unet_traced_filename.exists():
unet_traced_filename.parent.mkdir(parents=True, exist_ok=True)
unet_traced.save(unet_traced_filename)
if __name__ == "__main__":
jit_unet()