diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index e2ff15ca9..b80860f59 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -58,6 +58,7 @@ from nerfstudio.field_components.temporal_distortions import TemporalDistortionK from nerfstudio.models.depth_nerfacto import DepthNerfactoModelConfig from nerfstudio.models.dreamembedding import DreamEmbeddingModelConfig from nerfstudio.models.dreamfusion import DreamFusionModelConfig +from nerfstudio.models.iterative_dreamfusion import DreamFusionIterativeModelConfig from nerfstudio.models.instant_ngp import InstantNGPModelConfig from nerfstudio.models.mipnerf import MipNerfModel from nerfstudio.models.nerfacto import NerfactoModelConfig @@ -450,7 +451,7 @@ method_configs["iter_test"] = TrainerConfig( generative=True, datamanager=IterativeDataManagerConfig( ), - model=NerfactoModelConfig(eval_num_rays_per_chunk=1 << 15), + model=DreamFusionIterativeModelConfig(), ), optimizers={ "proposal_networks": { @@ -458,7 +459,7 @@ method_configs["iter_test"] = TrainerConfig( "scheduler": None, }, "fields": { - "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), + "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15), "scheduler": None, }, }, diff --git a/nerfstudio/data/datamanagers/iterative_datamanager.py b/nerfstudio/data/datamanagers/iterative_datamanager.py index 514d35b49..a060f65a9 100644 --- a/nerfstudio/data/datamanagers/iterative_datamanager.py +++ b/nerfstudio/data/datamanagers/iterative_datamanager.py @@ -28,8 +28,11 @@ import torch from rich.progress import Console from torch.nn import Parameter from typing_extensions import Literal +from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs +from nerfstudio.data.utils.nerfstudio_collate import nerfstudio_collate +from nerfstudio.model_components.ray_generators import RayGenerator from nerfstudio.cameras.cameras import Cameras from nerfstudio.cameras.rays import RayBundle @@ -37,7 +40,9 @@ from nerfstudio.configs.config_utils import to_immutable_dict from nerfstudio.data.datamanagers.base_datamanager import DataManager, DataManagerConfig from nerfstudio.data.datasets.base_dataset import InputDataset from nerfstudio.data.scene_box import SceneBox -from nerfstudio.data.utils.dataloaders import RandIndicesEvalDataloader +from nerfstudio.data.utils.dataloaders import RandIndicesEvalDataloader, CacheDataloader + +from nerfstudio.data.pixel_samplers import PixelSampler CONSOLE = Console(width=120) @@ -158,6 +163,17 @@ class IterativeDataManagerConfig(DataManagerConfig): """Configuration for data manager that does not load from a dataset. Instead, it generates random poses.""" _target: Type = field(default_factory=lambda: IterativeDataManager) + train_num_rays_per_batch: int = 1024 + """Number of rays per batch to use per training iteration.""" + train_num_images_to_sample_from: int = -1 + """Number of images to sample during training iteration.""" + train_num_times_to_repeat_images: int = -1 + """When not training on all images, number of iterations before picking new""" + collate_fn = staticmethod(nerfstudio_collate) + """Specifies the collate function to use for the train and eval dataloaders.""" + camera_optimizer: CameraOptimizerConfig = CameraOptimizerConfig() + """Specifies the camera pose optimizer used during training. Helpful if poses are noisy, such as for data from + Record3D.""" train_resolution: int = 64 """Training resolution""" eval_resolution: int = 64 @@ -217,7 +233,7 @@ class IterativeDataManager(DataManager): # pylint: disable=abstract-method radius_mean=self.config.radius_mean, radius_std=self.config.radius_std, focal_range=self.config.focal_range, - central_rotation_range=(-45, 45), + central_rotation_range=(-180, 180), vertical_rotation_range=self.config.vertical_rotation_range, jitter_std=self.config.jitter_std, center=self.config.center, @@ -232,24 +248,44 @@ class IterativeDataManager(DataManager): # pylint: disable=abstract-method num_workers=self.world_size * 4, ) + self.train_pixel_sampler = PixelSampler(self.config.train_num_rays_per_batch) + # pylint: disable=non-parent-init-called DataManager.__init__(self) - def create_dataset(self, filepaths: List[Path], cameras: Cameras): + def create_train_dataset(self, filepaths: List[Path], cameras: Cameras, camera_angles): dataparser_outputs = DataparserOutputs( image_filenames=filepaths, cameras=cameras, - scene_box=None, + scene_box=SceneBox(torch.Tensor([[-1, -1, -1], [1, 1, 1]])), mask_filenames=None, dataparser_transform=None, dataparser_scale=None, metadata=None, ) - return InputDataset(dataparser_outputs) + self.train_dataset = InputDataset(dataparser_outputs) + self.camera_angles = camera_angles + self.train_image_dataloader = CacheDataloader( + self.train_dataset, + num_images_to_sample_from=self.config.train_num_images_to_sample_from, + num_times_to_repeat_images=self.config.train_num_times_to_repeat_images, + device=self.device, + num_workers=self.world_size * 4, + pin_memory=True, + collate_fn=self.config.collate_fn, + ) + self.iter_train_image_dataloader = iter(self.train_image_dataloader) + self.train_camera_optimizer = self.config.camera_optimizer.setup( + num_cameras=self.train_dataset.cameras.size, device=self.device + ) + self.train_ray_generator = RayGenerator( + self.train_dataset.cameras.to(self.device), + self.train_camera_optimizer, + ) def random_train_views(self, step: int, resolution:int, num_views: int) -> InputDataset: """eawoijfaw""" - cameras, _, _ = random_train_pose( + cameras, vertical_rotation, central_rotation = random_train_pose( size=num_views, resolution=resolution, device= self.device, @@ -266,7 +302,7 @@ class IterativeDataManager(DataManager): # pylint: disable=abstract-method torch.tensor([[i] for i in range(num_views)]) ).flatten() - return ray_bundle, cameras + return ray_bundle, cameras, list(zip(vertical_rotation, central_rotation)) def next_train(self, step: int) -> Tuple[RayBundle, Dict]: """Returns the next batch of data from the train dataloader.""" diff --git a/nerfstudio/generative/stable_diffusion_img2img.py b/nerfstudio/generative/stable_diffusion_img2img.py index 0237adb96..ec9b96a94 100644 --- a/nerfstudio/generative/stable_diffusion_img2img.py +++ b/nerfstudio/generative/stable_diffusion_img2img.py @@ -32,6 +32,8 @@ from torch import nn from torch.cuda.amp import custom_bwd, custom_fwd from torch.cuda.amp.grad_scaler import GradScaler from torchtyping import TensorType +from PIL import Image + try: from diffusers import PNDMScheduler, StableDiffusionPipeline @@ -98,7 +100,7 @@ class StableDiffusionImg2Img(nn.Module): pipe = pipe.to(self.device) pipe.enable_attention_slicing() - pipe.enable_xformers_memory_efficient_attention() + # pipe.enable_xformers_memory_efficient_attention() # use jitted unet filename_sd_id = sd_id.split("/")[-1] @@ -283,7 +285,7 @@ class StableDiffusionImg2Img(nn.Module): self.scheduler.set_timesteps(num_inference_steps) # type: ignore with torch.autocast("cuda"): - for t in self.scheduler.timesteps: # type: ignore + for i, t in enumerate(self.scheduler.timesteps): # type: ignore assert latents is not None # expand the latents if we are doing classifier-free guidance to avoid doing two forward passes. latent_model_input = torch.cat([latents] * 2) @@ -300,6 +302,14 @@ class StableDiffusionImg2Img(nn.Module): # compute the previous noisy sample x_t -> x_t-1 latents = self.scheduler.step(noise_pred, t, latents)["prev_sample"] # type: ignore + + cur_img = self.latents_to_img(latents.half()) + cur_img = cur_img.detach().cpu().permute(0, 2, 3, 1).numpy() + cur_img = (cur_img * 255).round().astype("uint8")[0] # might delete this indexing + + save_path = Path(f"normal_diffusion_image_steps/{i}.png") + mediapy.write_image(str(save_path), cur_img) + return latents def latents_to_img(self, latents: TensorType["BS", 4, "H", "W"]) -> TensorType["BS", 3, "H", "W"]: @@ -333,24 +343,37 @@ class StableDiffusionImg2Img(nn.Module): return latents - def update_img(self, prompt, negative_prompt, image, num_inference_steps, guidance_scale): - prompt = [prompt] if isinstance(prompt, str) else prompt - negative_prompt = [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt - text_embeddings = self.get_text_embeds(prompt, negative_prompt) + def update_img(self, text_embedding, image, num_inference_steps=50, guidance_scale=7.5): + self.scheduler.set_timesteps(50) # type: ignore + latent = self.imgs_to_latent(image.half()) - # image = F.interpolate(image.half(), (IMG_DIM, IMG_DIM), mode="bilinear") - init_latents = self.imgs_to_latent(image.half()) + # t = torch.randint(self.max_step - 1, self.max_step, [1], dtype=torch.long, device=self.device) + t = torch.randint(0, 1, [1], dtype=torch.long) + new_steps = self.scheduler.timesteps[t:] - latents = self.produce_latents( - text_embeddings, - height=IMG_DIM, - width=IMG_DIM, - latents=init_latents, - num_inference_steps=num_inference_steps, - guidance_scale=guidance_scale, - ) # [1, 4, resolution, resolution] + t_noise = self.scheduler.timesteps[t] - diffused_img = self.latents_to_img(latents.half()) + noise = torch.randn_like(latent) + latent = self.scheduler.add_noise(latent, noise, t_noise) # type: ignore + + # for i, time in enumerate(self.scheduler.timesteps): + for i, time in enumerate(new_steps): + # for i in range(num_inference_steps): + # predict the noise residual + latent_model_input = torch.cat([latent] * 2) + + with torch.no_grad(): + noise_pred = self.unet( + latent_model_input, time.to(self.device), encoder_hidden_states=text_embedding + ).sample + + # perform guidance + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_text + guidance_scale * (noise_pred_text - noise_pred_uncond) + + latent = self.scheduler.step(noise_pred, time, latent)["prev_sample"] # type: ignore + + diffused_img = self.latents_to_img(latent.half()) # diffused_img = F.interpolate(diffused_img, (128, 128), mode="bilinear") diffused_img = diffused_img.detach().cpu().permute(0, 2, 3, 1).numpy() diffused_img = (diffused_img * 255).round().astype("uint8")[0] # might delete this indexing @@ -395,7 +418,7 @@ class StableDiffusionImg2Img(nn.Module): return diffused_img def forward( - self, prompts, negative_prompts="", num_inference_steps=50, guidance_scale=7.5, latents=None + self, prompts, negative_prompts="", num_inference_steps=150, guidance_scale=7.5, latents=None ) -> np.ndarray: """Generate an image from a prompt. Args: @@ -411,7 +434,7 @@ class StableDiffusionImg2Img(nn.Module): def generate_image( - prompt: str, negative: str = "", seed: int = 0, steps: int = 50, save_path: Path = Path("test_sd.png") + prompt: str, negative: str = "", image_path: str = "", seed: int = 0, steps: int = 50, save_path: Path = Path("test_sd.png") ): """Generate an image from a prompt using Stable Diffusion. Args: @@ -425,9 +448,17 @@ def generate_image( torch.cuda.manual_seed(seed) cuda_device = torch.device("cuda") with torch.no_grad(): - sd = StableDiffusion(cuda_device) - imgs = sd.prompt_to_img(prompt, negative, steps) - mediapy.write_image(str(save_path), imgs[0]) + sd = StableDiffusionImg2Img(cuda_device) + pil_image = Image.open(image_path) + image = np.array(pil_image, dtype="uint8") # shape is (h, w) or (h, w, 3 or 4) + image = torch.from_numpy(image.astype("float32") / 255.0) + image = image[None, :, :, :3].permute(0, 3, 1, 2).to(cuda_device) + + num_steps=50 + + imgs = sd.update_img(prompt, negative, image, num_steps) + # imgs = sd.prompt_to_img(prompt, negative) + mediapy.write_image(str(save_path), imgs) if __name__ == "__main__": diff --git a/nerfstudio/models/iterative_dreamfusion.py b/nerfstudio/models/iterative_dreamfusion.py new file mode 100644 index 000000000..942e86387 --- /dev/null +++ b/nerfstudio/models/iterative_dreamfusion.py @@ -0,0 +1,327 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +DreamFusion implementation. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Tuple, Type + +import numpy as np +import torch +from torch.nn import Parameter +from typing_extensions import Literal + +from nerfstudio.cameras.rays import RayBundle +from nerfstudio.engine.callbacks import ( + TrainingCallback, + TrainingCallbackAttributes, + TrainingCallbackLocation, +) +from nerfstudio.field_components.field_heads import FieldHeadNames +from nerfstudio.fields.density_fields import HashMLPDensityField +from nerfstudio.fields.dreamfusion_field import DreamFusionField +from nerfstudio.generative.stable_diffusion import StableDiffusion +from nerfstudio.generative.stable_diffusion_utils import PositionalTextEmbeddings +from nerfstudio.model_components.losses import ( + MSELoss, + distortion_loss, + interlevel_loss, + orientation_loss, + pred_normal_loss, +) +from nerfstudio.model_components.ray_samplers import ( + ProposalNetworkSampler, + UniformSampler, +) +from nerfstudio.model_components.renderers import ( + AccumulationRenderer, + DepthRenderer, + NormalsRenderer, + RGBRenderer, +) +from nerfstudio.model_components.scene_colliders import AABBBoxCollider, SphereCollider +from nerfstudio.model_components.shaders import LambertianShader, NormalsShader +from nerfstudio.models.base_model import Model, ModelConfig +from nerfstudio.utils import colormaps, colors, math, misc + + +@dataclass +class DreamFusionIterativeModelConfig(ModelConfig): + """DreamFusion model config""" + + _target: Type = field(default_factory=lambda: DreamFusionIterativeModel) + """target class to instantiate""" + + """Strength schedule of center density""" + sphere_collider: bool = True + """Use spherical collider instead of box""" + + initialize_density: bool = True + """Initialize density in center of scene.""" + taper_range: Tuple[int, int] = (0, 5000) + """Range of step values for the density tapering""" + taper_strength: Tuple[float, float] = (1.0, 0.0) + """Strength schedule of center density""" + + """target transmittance for opacity penalty. This is the percent of the scene that is + background when rendered at the end of training""" + transmittance_end_schedule: int = 1500 + """number of iterations to reach target_transmittance_end""" + num_proposal_samples_per_ray: Tuple[int, ...] = (256, 96) + """Number of samples per ray for each proposal network.""" + num_nerf_samples_per_ray: int = 48 + """Number of samples per ray for the nerf network.""" + proposal_update_every: int = 5 + """Sample every n steps after the warmup""" + proposal_warmup: int = 5000 + """Scales n from 1 to proposal_update_every over this many steps""" + num_proposal_iterations: int = 2 + """Number of proposal network iterations.""" + use_same_proposal_network: bool = False + """Use the same proposal network. Otherwise use different ones.""" + proposal_net_args_list: List[Dict] = field( + default_factory=lambda: [ + {"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 128}, + {"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 256}, + ] + ) + """Arguments for the proposal density fields.""" + proposal_weights_anneal_slope: float = 10.0 + """Slope of the annealing function for the proposal weights.""" + proposal_weights_anneal_max_num_iters: int = 500 + """Max num iterations for the annealing function.""" + use_single_jitter: bool = True + """Whether use single jitter or not for the proposal networks.""" + interlevel_loss_mult: float = 1.0 + """Proposal loss multiplier.""" + distortion_loss_mult: float = 1.0 + """scale for opacity penalty""" + max_res: int = 256 + """Maximum resolution of the density field.""" + + + +class DreamFusionIterativeModel(Model): + """DreamFusionModel Model + + Args: + config: DreamFusion configuration to instantiate model + """ + + config: DreamFusionIterativeModelConfig + + def __init__( + self, + config: DreamFusionIterativeModelConfig, + **kwargs, + ) -> None: + self.initialize_density = config.initialize_density + self.density_strength = 5 + super().__init__(config=config, **kwargs) + + def populate_modules(self): + """Set the fields and modules""" + super().populate_modules() + + # setting up fields + self.field = DreamFusionField(self.scene_box.aabb, max_res=self.config.max_res) + + # samplers + self.density_fns = [] + num_prop_nets = self.config.num_proposal_iterations + # Build the proposal network(s) + self.proposal_networks = torch.nn.ModuleList() + + for i in range(num_prop_nets): + prop_net_args = self.config.proposal_net_args_list[min(i, len(self.config.proposal_net_args_list) - 1)] + network = HashMLPDensityField( + self.scene_box.aabb, + **prop_net_args, + ) + self.proposal_networks.append(network) + self.density_fns.extend([network.density_fn for network in self.proposal_networks]) + + update_schedule = lambda step: np.clip( + np.interp(step, [0, self.config.proposal_warmup], [0, self.config.proposal_update_every]), + 1, + self.config.proposal_update_every, + ) + self.proposal_sampler = ProposalNetworkSampler( + num_nerf_samples_per_ray=self.config.num_nerf_samples_per_ray, + num_proposal_samples_per_ray=self.config.num_proposal_samples_per_ray, + num_proposal_network_iterations=self.config.num_proposal_iterations, + single_jitter=self.config.use_single_jitter, + update_sched=update_schedule, + initial_sampler=UniformSampler(single_jitter=self.config.use_single_jitter), + pdf_histogram_padding=0.005, + ) + + # renderers + self.renderer_rgb = RGBRenderer(background_color=colors.WHITE) + self.renderer_accumulation = AccumulationRenderer() + self.renderer_depth = DepthRenderer() + self.renderer_normals = NormalsRenderer() + + # shaders + self.shader_lambertian = LambertianShader() + self.shader_normals = NormalsShader() + + # losses + self.rgb_loss = MSELoss() + + # colliders + if self.config.sphere_collider: + self.collider = SphereCollider(torch.Tensor([0, 0, 0]), 1.0) + else: + self.collider = AABBBoxCollider(scene_box=self.scene_box) + + def get_training_callbacks( + self, training_callback_attributes: TrainingCallbackAttributes + ) -> List[TrainingCallback]: + + # the callback that we want to run every X iterations after the training iteration + def taper_density( + self, training_callback_attributes: TrainingCallbackAttributes, step: int # pylint: disable=unused-argument + ): + self.density_strength = np.interp(step, self.config.taper_range, self.config.taper_strength) + + + # anneal the weights of the proposal network before doing PDF sampling + def set_anneal(step): + # https://arxiv.org/pdf/2111.12077.pdf eq. 18 + train_frac = np.clip(step / self.config.proposal_weights_anneal_max_num_iters, 0, 1) + bias = lambda x, b: (b * x) / ((b - 1) * x + 1) + anneal = bias(train_frac, self.config.proposal_weights_anneal_slope) + self.proposal_sampler.set_anneal(anneal) + + callbacks = [ + TrainingCallback( + where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION], + func=taper_density, + update_every_num_iters=1, + args=[self, training_callback_attributes], + ), + TrainingCallback( + where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION], + update_every_num_iters=1, + func=self.proposal_sampler.step_cb, + ), + TrainingCallback( + where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION], + update_every_num_iters=1, + func=set_anneal, + ), + ] + return callbacks + + def get_param_groups(self) -> Dict[str, List[Parameter]]: + param_groups = {} + param_groups["proposal_networks"] = list(self.proposal_networks.parameters()) + param_groups["fields"] = list(self.field.parameters()) + return param_groups + + def get_outputs(self, ray_bundle: RayBundle): # pylint: disable=too-many-statements + # uniform sampling + background_rgb = self.field.get_background_rgb(ray_bundle) + ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns) + field_outputs = self.field(ray_samples, compute_normals=True) + density = field_outputs[FieldHeadNames.DENSITY] + + if self.initialize_density: + pos = ray_samples.frustums.get_positions() + density_blob = self.density_strength * torch.exp(-torch.norm(pos, dim=-1) / (2 * 0.04))[..., None] + density = density + density_blob + + weights = ray_samples.get_weights(density) + weights_list.append(weights) + ray_samples_list.append(ray_samples) + + accumulation = self.renderer_accumulation(weights) + depth = self.renderer_depth(weights, ray_samples) + rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights) + + accum_mask = torch.clamp((torch.nan_to_num(accumulation, nan=0.0)), min=0.0, max=1.0) + accum_mask_inv = 1.0 - accum_mask + + background = accum_mask_inv * background_rgb + + outputs = { + "background_rgb": background_rgb, + "background": background, + "accumulation": accum_mask, + "depth": depth, + } + + # These use a lot of GPU memory, so we avoid storing them for eval. + if self.training: + outputs["weights_list"] = weights_list + outputs["ray_samples_list"] = ray_samples_list + + for i in range(self.config.num_proposal_iterations): + outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i]) + + outputs["rgb"] = rgb + + return outputs + + def get_loss_dict(self, outputs, batch, metrics_dict=None) -> Dict[str, torch.Tensor]: + # Scaling metrics by coefficients to create the losses. + + loss_dict = {} + loss_dict = misc.scale_dict(loss_dict, self.config.loss_coefficients) + + image = batch["image"].to(self.device) + loss_dict["rgb_loss"] = self.rgb_loss(image, outputs["rgb"]) + + if self.training: + loss_dict["distortion_loss"] = self.config.distortion_loss_mult * distortion_loss( + outputs["weights_list"], outputs["ray_samples_list"] + ) + loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss( + outputs["weights_list"], outputs["ray_samples_list"] + ) + return loss_dict + + def get_image_metrics_and_images( + self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor] + ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: + acc = colormaps.apply_colormap(outputs["accumulation"]) + depth = colormaps.apply_depth_colormap( + outputs["depth"], + accumulation=outputs["accumulation"], + ) + prop_depth_0 = colormaps.apply_depth_colormap( + outputs["prop_depth_0"], + accumulation=outputs["accumulation"], + ) + prop_depth_1 = colormaps.apply_depth_colormap( + outputs["prop_depth_1"], + accumulation=outputs["accumulation"], + ) + + metrics_dict = {} + images_dict = { + "img": outputs["rgb"], + "accumulation": acc, + "depth": depth, + "prop_depth_0": prop_depth_0, + "prop_depth_1": prop_depth_1, + "normals": outputs["normals"], + "pred_normals": outputs["pred_normals"], + } + return metrics_dict, images_dict diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py index 22405f60c..62e6c12a5 100644 --- a/nerfstudio/pipelines/base_pipeline.py +++ b/nerfstudio/pipelines/base_pipeline.py @@ -127,7 +127,7 @@ class Pipeline(nn.Module): assert self.datamanager.train_sampler is not None self.datamanager.train_sampler.set_epoch(step) ray_bundle, batch = self.datamanager.next_train(step) - model_outputs = self.model(ray_bundle, batch) + model_outputs = self.model(ray_bundle) metrics_dict = self.model.get_metrics_dict(model_outputs, batch) loss_dict = self.model.get_loss_dict(model_outputs, batch, metrics_dict) diff --git a/nerfstudio/pipelines/iterative_pipeline.py b/nerfstudio/pipelines/iterative_pipeline.py index 933eb81f5..24d142b6a 100644 --- a/nerfstudio/pipelines/iterative_pipeline.py +++ b/nerfstudio/pipelines/iterative_pipeline.py @@ -26,6 +26,7 @@ from torch.cuda.amp.grad_scaler import GradScaler from typing_extensions import Literal import torch.distributed as dist import mediapy +from nerfstudio.generative.stable_diffusion_utils import PositionalTextEmbeddings from nerfstudio.data.datamanagers.base_datamanager import ( @@ -57,7 +58,7 @@ class IterativePipelineConfig(VanillaPipelineConfig): """specifies whether the pipeline is for generative models""" temp_save_path: Path=PurePath("") """save path for generated dataset""" - prompt: str="a pineapple" + prompt: str="a photo of a pineapple" class IterativePipeline(VanillaPipeline): """Pipeline with logic for saving outputs from nerf for use as training images.""" @@ -76,10 +77,18 @@ class IterativePipeline(VanillaPipeline): self.datamanager: IterativeDataManager = config.datamanager.setup( device=device, test_mode=test_mode, world_size=world_size, local_rank=local_rank ) - self.train_dataset = None sd_version = "1-5" - self.prompt = config.prompt + self.sd = StableDiffusionImg2Img(self.device, version=sd_version) + self.text_embeddings = PositionalTextEmbeddings( + base_prompt=config.prompt, + top_prompt=config.prompt + ", overhead view", + side_prompt=config.prompt + ", side view", + back_prompt=config.prompt + ", back view", + front_prompt=config.prompt + ", front view", + stable_diffusion=self.sd, + positional_prompting="discrete", + ) def forward(self): @@ -92,14 +101,12 @@ class IterativePipeline(VanillaPipeline): @torch.no_grad() def save_new_train_images(self, step: int): print("saving training images") - num_images_per_train = 5 + num_images_per_train = 7 resolution = 512 - ray_bundles, cameras = self.datamanager.random_train_views(step, resolution=resolution, num_views=num_images_per_train) + ray_bundles, cameras, camera_angles = self.datamanager.random_train_views(step, resolution=resolution, num_views=num_images_per_train) train_img_paths = [] num_rays_per_chunk = 512 - self.model.eval() - for i in range(num_images_per_train): total_rgb = [] for j in range(0, resolution * resolution, num_rays_per_chunk): @@ -115,18 +122,23 @@ class IterativePipeline(VanillaPipeline): train_img_paths.append(str(save_path)) mediapy.write_image(str(save_path), img) - self.train_dataset = self.datamanager.create_dataset(train_img_paths, cameras) + self.datamanager.create_train_dataset(train_img_paths, cameras, camera_angles) @torch.no_grad() - def update_train_images(self): + def update_train_images(self, step: int): print("updating training images") - for i in range(len(self.train_dataset)): - # for i in range(1): - image = self.train_dataset.get_image(i)[None, :, :, :].permute(0, 3, 1, 2).to(self.sd.device) - print(i) - image = self.sd.update_img(self.prompt, "", image, num_inference_steps=50, guidance_scale=7.5) - mediapy.write_image(self.train_dataset.image_filenames[i], image) - del image + for i in range(len(self.datamanager.train_dataset)): + camera_angles = self.datamanager.camera_angles[i] + text_embedding = self.text_embeddings.get_text_embedding(camera_angles[0], camera_angles[1]) + + image = self.datamanager.train_dataset.get_image(i)[None, :, :, :].permute(0, 3, 1, 2).to(self.sd.device) + image = self.sd.update_img(text_embedding, image, num_inference_steps=50, guidance_scale=7.5) + mediapy.write_image(self.datamanager.train_dataset.image_filenames[i], image) + + filenames = self.datamanager.train_dataset.image_filenames + cameras = self.datamanager.train_dataset.cameras + + self.datamanager.create_train_dataset(filenames, cameras, self.datamanager.camera_angles) @profiler.time_function def get_train_loss_dict(self, step: int): @@ -136,21 +148,19 @@ class IterativePipeline(VanillaPipeline): # use a data loader that allows for multiview supervision each train step # rerender train dataset every Y iterations - if step == 0: + if step % 2000 == 0: self.save_new_train_images(step) - # snapshot1 = tracemalloc.take_snapshot() + if step % 500 == 0: + self.update_train_images(step) - if step % 20 == 0: - self.update_train_images() + if self.world_size > 1 and step: + assert self.datamanager.train_sampler is not None + self.datamanager.train_sampler.set_epoch(step) - # snapshot2 = tracemalloc.take_snapshot() - # top_stats = snapshot2.compare_to(snapshot1, 'lineno') + ray_bundle, batch = self.datamanager.next_train(step) + model_outputs = self.model(ray_bundle) + metrics_dict = self.model.get_metrics_dict(model_outputs, batch) + loss_dict = self.model.get_loss_dict(model_outputs, batch, metrics_dict) - # print("[ Top 10 differences ]") - # for stat in top_stats[:10]: - # print(stat) - - raise Exception - - return \ No newline at end of file + return model_outputs, loss_dict, metrics_dict \ No newline at end of file