From 8ba8942779f2456b49b9c1e9464bda1771a11c3a Mon Sep 17 00:00:00 2001 From: terrance Date: Fri, 3 Feb 2023 01:42:05 +0000 Subject: [PATCH] change uniform sampler to prediction network, add option to jit unet for faster sds_loss --- nerfstudio/generative/stable_diffusion.py | 34 +++- nerfstudio/models/dreamfusion.py | 172 ++++++++++++++----- nerfstudio/pipelines/dreamfusion_pipeline.py | 6 +- scripts/trace_unet.py | 83 +++++++++ 4 files changed, 248 insertions(+), 47 deletions(-) create mode 100644 scripts/trace_unet.py diff --git a/nerfstudio/generative/stable_diffusion.py b/nerfstudio/generative/stable_diffusion.py index af3aa2196..eb16710f0 100644 --- a/nerfstudio/generative/stable_diffusion.py +++ b/nerfstudio/generative/stable_diffusion.py @@ -17,9 +17,11 @@ # Modified from https://github.com/ashawkey/stable-dreamfusion/blob/main/nerf/sd.py import sys +from dataclasses import dataclass from pathlib import Path from typing import List, Optional, Tuple, Union +import appdirs import mediapy import numpy as np import torch @@ -49,6 +51,11 @@ SD_SOURCE = "runwayml/stable-diffusion-v1-5" CLIP_SOURCE = "openai/clip-vit-large-patch14" +@dataclass +class UNet2DConditionOutput: + sample: torch.FloatTensor + + class StableDiffusion(nn.Module): """Stable Diffusion implementation @@ -81,13 +88,34 @@ class StableDiffusion(nn.Module): pipe.enable_attention_slicing() + # use jitted unet + unet_traced_filename = Path(appdirs.user_data_dir("nerfstudio")) / "unet_traced.pt" + if unet_traced_filename.exists(): + CONSOLE.print("Loading traced UNet.") + unet_traced = torch.jit.load("unet_traced.pt") + + class TracedUNet(torch.nn.Module): + """Jitted version of UNet""" + + def __init__(self): + super().__init__() + self.in_channels = pipe.unet.in_channels + self.device = pipe.unet.device + + def forward(self, latent_model_input, t, encoder_hidden_states): + sample = unet_traced(latent_model_input, t, encoder_hidden_states)[0] + return UNet2DConditionOutput(sample=sample) + + self.unet = TracedUNet() + del pipe.unet + else: + self.unet = pipe.unet + self.unet.to(memory_format=torch.channels_last) + self.tokenizer = pipe.tokenizer - self.unet = pipe.unet self.text_encoder = pipe.text_encoder self.auto_encoder = pipe.vae - self.unet.to(memory_format=torch.channels_last) - CONSOLE.print("Stable Diffusion loaded!") def get_text_embeds( diff --git a/nerfstudio/models/dreamfusion.py b/nerfstudio/models/dreamfusion.py index df617942e..0ebeef019 100644 --- a/nerfstudio/models/dreamfusion.py +++ b/nerfstudio/models/dreamfusion.py @@ -37,13 +37,18 @@ from nerfstudio.engine.callbacks import ( # from nerfstudio.field_components.encodings import NeRFEncoding, from nerfstudio.field_components.field_heads import FieldHeadNames +from nerfstudio.fields.density_fields import HashMLPDensityField from nerfstudio.fields.dreamfusion_field import DreamFusionField from nerfstudio.model_components.losses import ( MSELoss, + interlevel_loss, orientation_loss, pred_normal_loss, ) -from nerfstudio.model_components.ray_samplers import UniformSampler +from nerfstudio.model_components.ray_samplers import ( + ProposalNetworkSampler, + UniformSampler, +) from nerfstudio.model_components.renderers import ( AccumulationRenderer, DepthRenderer, @@ -85,6 +90,33 @@ class DreamFusionModelConfig(ModelConfig): background when rendered at the end of training""" transmittance_end_schedule: int = 1000 """number of iterations to reach target_transmittance_end""" + num_proposal_samples_per_ray: Tuple[int, ...] = (256, 96) + """Number of samples per ray for each proposal network.""" + num_nerf_samples_per_ray: int = 48 + """Number of samples per ray for the nerf network.""" + proposal_update_every: int = 5 + """Sample every n steps after the warmup""" + proposal_warmup: int = 5000 + """Scales n from 1 to proposal_update_every over this many steps""" + num_proposal_iterations: int = 2 + """Number of proposal network iterations.""" + use_same_proposal_network: bool = False + """Use the same proposal network. Otherwise use different ones.""" + proposal_net_args_list: List[Dict] = field( + default_factory=lambda: [ + {"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 128}, + {"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 256}, + ] + ) + """Arguments for the proposal density fields.""" + proposal_weights_anneal_slope: float = 10.0 + """Slope of the annealing function for the proposal weights.""" + proposal_weights_anneal_max_num_iters: int = 1000 + """Max num iterations for the annealing function.""" + use_single_jitter: bool = True + """Whether use single jitter or not for the proposal networks.""" + interlevel_loss_mult: float = 1.0 + """Proposal loss multiplier.""" class DreamFusionModel(Model): @@ -109,6 +141,64 @@ class DreamFusionModel(Model): self.target_transmittance = config.target_transmittance_start super().__init__(config=config, **kwargs) + def populate_modules(self): + """Set the fields and modules""" + super().populate_modules() + + # setting up fields + self.field = DreamFusionField(self.scene_box.aabb) + + # samplers + self.density_fns = [] + num_prop_nets = self.config.num_proposal_iterations + # Build the proposal network(s) + self.proposal_networks = torch.nn.ModuleList() + + for i in range(num_prop_nets): + prop_net_args = self.config.proposal_net_args_list[min(i, len(self.config.proposal_net_args_list) - 1)] + network = HashMLPDensityField( + self.scene_box.aabb, + **prop_net_args, + ) + self.proposal_networks.append(network) + self.density_fns.extend([network.density_fn for network in self.proposal_networks]) + + update_schedule = lambda step: np.clip( + np.interp(step, [0, self.config.proposal_warmup], [0, self.config.proposal_update_every]), + 1, + self.config.proposal_update_every, + ) + self.proposal_sampler = ProposalNetworkSampler( + num_nerf_samples_per_ray=self.config.num_nerf_samples_per_ray, + num_proposal_samples_per_ray=self.config.num_proposal_samples_per_ray, + num_proposal_network_iterations=self.config.num_proposal_iterations, + single_jitter=self.config.use_single_jitter, + update_sched=update_schedule, + ) + + # self.sampler_uniform = UniformSampler(num_samples=self.num_samples, single_jitter=True) + + # renderers + self.renderer_rgb = RGBRenderer(background_color=colors.WHITE) + self.renderer_accumulation = AccumulationRenderer() + self.renderer_depth = DepthRenderer() + self.renderer_normals = NormalsRenderer() + self.renderer_lambertian = LambertianShadingRenderer() + + # losses + self.rgb_loss = MSELoss() + + # metrics + self.psnr = PeakSignalNoiseRatio(data_range=1.0) + self.ssim = structural_similarity_index_measure + self.lpips = LearnedPerceptualImagePatchSimilarity() + + # colliders + if self.config.sphere_collider: + self.collider = SphereCollider(torch.Tensor([0, 0, 0]), 1.0) + else: + self.collider = AABBBoxCollider(scene_box=self.scene_box) + def get_training_callbacks( self, training_callback_attributes: TrainingCallbackAttributes ) -> List[TrainingCallback]: @@ -137,6 +227,16 @@ class DreamFusionModel(Model): self.config.target_transmittance_start - self.config.target_transmittance_end ) + # anneal the weights of the proposal network before doing PDF sampling + N = self.config.proposal_weights_anneal_max_num_iters + + def set_anneal(step): + # https://arxiv.org/pdf/2111.12077.pdf eq. 18 + train_frac = np.clip(step / N, 0, 1) + bias = lambda x, b: (b * x) / ((b - 1) * x + 1) + anneal = bias(train_frac, self.config.proposal_weights_anneal_slope) + self.proposal_sampler.set_anneal(anneal) + callbacks = [ TrainingCallback( where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION], @@ -162,41 +262,19 @@ class DreamFusionModel(Model): func=update_target_transmittance, args=[self, training_callback_attributes], ), + TrainingCallback( + where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION], + update_every_num_iters=1, + func=self.proposal_sampler.step_cb, + ), + TrainingCallback( + where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION], + update_every_num_iters=1, + func=set_anneal, + ), ] return callbacks - def populate_modules(self): - """Set the fields and modules""" - super().populate_modules() - - # setting up fields - - self.field = DreamFusionField(self.scene_box.aabb) - - # samplers - self.sampler_uniform = UniformSampler(num_samples=self.num_samples, single_jitter=True) - - # renderers - self.renderer_rgb = RGBRenderer(background_color=colors.WHITE) - self.renderer_accumulation = AccumulationRenderer() - self.renderer_depth = DepthRenderer() - self.renderer_normals = NormalsRenderer() - self.renderer_lambertian = LambertianShadingRenderer() - - # losses - self.rgb_loss = MSELoss() - - # metrics - self.psnr = PeakSignalNoiseRatio(data_range=1.0) - self.ssim = structural_similarity_index_measure - self.lpips = LearnedPerceptualImagePatchSimilarity() - - # colliders - if self.config.sphere_collider: - self.collider = SphereCollider(torch.Tensor([0, 0, 0]), 1.0) - else: - self.collider = AABBBoxCollider(scene_box=self.scene_box) - def get_param_groups(self) -> Dict[str, List[Parameter]]: param_groups = {} param_groups["fields"] = list(self.field.parameters()) @@ -205,13 +283,16 @@ class DreamFusionModel(Model): def get_outputs(self, ray_bundle: RayBundle): # uniform sampling background_rgb = self.field.get_background_rgb(ray_bundle) - # ray_bundle = self.sphere_collider(ray_bundle) - ray_samples_uniform = self.sampler_uniform(ray_bundle) - field_outputs = self.field(ray_samples_uniform, compute_normals=True) + ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns) + field_outputs = self.field(ray_samples, compute_normals=True) + weights = ray_samples.get_weights(field_outputs[FieldHeadNames.DENSITY]) + weights_list.append(weights) + ray_samples_list.append(ray_samples) density = field_outputs[FieldHeadNames.DENSITY] + if self.initialize_density and self.training: - pos = ray_samples_uniform.frustums.get_positions() + pos = ray_samples.frustums.get_positions() density_blob = ( self.config.init_density_strength * self.density_strength @@ -219,10 +300,8 @@ class DreamFusionModel(Model): ) density += density_blob - weights = ray_samples_uniform.get_weights(density) - accumulation = self.renderer_accumulation(weights) - depth = self.renderer_depth(weights, ray_samples_uniform) + depth = self.renderer_depth(weights, ray_samples) rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights) accum_mask = torch.clamp((torch.nan_to_num(accumulation, nan=0.0)), min=0.0, max=1.0) @@ -238,6 +317,14 @@ class DreamFusionModel(Model): "depth": depth, } + # These use a lot of GPU memory, so we avoid storing them for eval. + if self.training: + outputs["weights_list"] = weights_list + outputs["ray_samples_list"] = ray_samples_list + + for i in range(self.config.num_proposal_iterations): + outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i]) + normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights) pred_normals = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights) # .detach() @@ -313,7 +400,10 @@ class DreamFusionModel(Model): loss_dict["opacity_loss"] = -torch.minimum( (1 - outputs["accumulation"]).mean(), torch.tensor(self.target_transmittance) ) - + if self.training: + loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss( + outputs["weights_list"], outputs["ray_samples_list"] + ) return loss_dict def get_image_metrics_and_images( diff --git a/nerfstudio/pipelines/dreamfusion_pipeline.py b/nerfstudio/pipelines/dreamfusion_pipeline.py index cfb8a967c..a2ba3558b 100644 --- a/nerfstudio/pipelines/dreamfusion_pipeline.py +++ b/nerfstudio/pipelines/dreamfusion_pipeline.py @@ -132,9 +132,9 @@ class DreamfusionPipeline(VanillaPipeline): ) loss_dict["sds_loss"] = sds_loss.to(self.device) # TODO: opacity penalty using transmittance, not accumultation - if self.config.opacity_penalty: - accum_mean = np.mean(1.0 - accumulation) - sds_loss *= np.min((0.5, accum_mean)) + # if self.config.opacity_penalty: + # accum_mean = np.mean(1.0 - accumulation) + # sds_loss *= np.min((0.5, accum_mean)) model_outputs["latents"] = latents model_outputs["grad"] = grad diff --git a/scripts/trace_unet.py b/scripts/trace_unet.py new file mode 100644 index 000000000..17577037b --- /dev/null +++ b/scripts/trace_unet.py @@ -0,0 +1,83 @@ +"""Trace Stable Diffusion UNet for speed improvement""" + +import functools +import time +from pathlib import Path + +import appdirs +import torch +from diffusers import StableDiffusionPipeline + + +def jit_unet(): + + # torch disable grad + torch.set_grad_enabled(False) + + # set variables + n_experiments = 2 + unet_runs_per_experiment = 50 + + # load inputs + def generate_inputs(): + sample = torch.randn(2, 4, 64, 64).half().cuda() + timestep = torch.rand(1).half().cuda() * 999 + encoder_hidden_states = torch.randn(2, 77, 768).half().cuda() + return sample, timestep, encoder_hidden_states + + pipe = StableDiffusionPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", + torch_dtype=torch.float16, + ).to("cuda") + + pipe.enable_attention_slicing() + + unet = pipe.unet + unet.eval() + unet.to(memory_format=torch.channels_last) # use channels_last memory format + unet.forward = functools.partial(unet.forward, return_dict=False) # set return_dict=False as default + + # warmup + for _ in range(3): + with torch.inference_mode(): + inputs = generate_inputs() + orig_output = unet(*inputs) + + # trace + print("tracing..") + unet_traced = torch.jit.trace(unet, inputs) + unet_traced.eval() + print("done tracing") + + # warmup and optimize graph + for _ in range(5): + with torch.inference_mode(): + inputs = generate_inputs() + orig_output = unet_traced(*inputs) + + # benchmarking + with torch.inference_mode(): + for _ in range(n_experiments): + torch.cuda.synchronize() + start_time = time.time() + for _ in range(unet_runs_per_experiment): + orig_output = unet_traced(*inputs) + torch.cuda.synchronize() + print(f"unet traced inference took {time.time() - start_time:.2f} seconds") + for _ in range(n_experiments): + torch.cuda.synchronize() + start_time = time.time() + for _ in range(unet_runs_per_experiment): + orig_output = unet(*inputs) + torch.cuda.synchronize() + print(f"unet inference took {time.time() - start_time:.2f} seconds") + + # save the traced model + unet_traced_filename = Path(appdirs.user_data_dir("nerfstudio")) / "unet_traced.pt" + if not unet_traced_filename.exists(): + unet_traced_filename.parent.mkdir(parents=True, exist_ok=True) + unet_traced.save(unet_traced_filename) + + +if __name__ == "__main__": + jit_unet()