From b17268e83b38546dd984c398905e82c9c641db6a Mon Sep 17 00:00:00 2001 From: Terrance Wang Date: Sat, 17 Jun 2023 23:34:04 -0700 Subject: [PATCH] [WIP] dreamfusion implementation (#1008) * starting the framework for dreamfusion * Removed view direction from RGB to calculate albedo * Started diffusion * Added dreamfusion to CLI, removed direction embeddings * Normals in dreamfusion * Lambertian shading added * Train normals in Dreamfusion * Stable diffusion from Stable Dreamfusion * Fix normal orientation loss * Configurable light direction sampling * adding density initalization at origin * V1 datamanager * Don't train normals for first 1k iters * SDS loss added to dreamfusion * Dreamfusion pipeline * Background MLP and SphereCollider * Normals prediction training fixed * Sigmoid on background, prompt in config * Shading hyperparameter adjustements * Shading hyperparameter tuning * Location based prompting, density initialization fades, LR change * Cleaned imports, more config options * Tuning, more config changes * remove unused sampler, normalize predicted normals from network * change position encoding to direction encoding for background color mlp * Updating dreamfusion to use custom trainer, fixed bug, multi-gpu support (put diffusion on one specific device), merged with main * Hard coding less, offloading to config * Appeasing the linter * First pass opacity / sparsity losses * Small patches and configs * Accidentally pushed logs * Removing logs for real this time * Transmittance loss bug fix * 12 GB Dreamfusion! * Lint * Remove unecissary collider changes * Start SD cleanup * Cleanup SD code * adding overhead location prompting * Use lambertian renderer * turn grad accum off by default, decrease background mlp size, add overhead text embedding * Cleanup * Fix normals vis * change uniform sampler to prediction network, add option to jit unet for faster sds_loss * Use shaders * cleaning up jit code * add warning if using non jit unet * add proposal network optimizer to config * speed up render and cleanup code * Clean up datamanager * Add distortion loss * fix wanb * adding interpolated location sampling * Update configs * interpolated location prompting option, lowering init density, transmittance loss interpolation flipped * rename alphas mult to alphas loss mult * Mess around with normals * add option to use random bg color + bg mlp * Remove Dreamfusion Trainer (#1423) * Dreamfusion trainer no longer needed * Added typing for grad scaler * SDS return loss * Fix CFG * Cleanup * Update configs * Fix opacity loss, use unform sampling first round of prop * add prompt edditing to viewer * Choose your own diffusion * remove dreamfusion pipeline file * merging + remove dreamfusion pipeline file * Update configs * add stable diff utils * cleanup * cleanup * Fix test * positional prompting bugfix * Warmups * tune * lint * Remove dataparser * merging + text embedding bugfix * Set data path to launch viewer for DreamFusion (#1776) * Set data path to launch viewer for DreamFusion * Apply formatter * Fallback to 'base_dir' if data manager has no path * Make if statement non-inline * Reformat --------- Co-authored-by: Matthew Tancik * add deepfloyd option first try * merging * merging from main * merging * removing comments, cleaning code, making stable diff utils usable for deepfloyd, remove deepfloyd text encoder after use * undo chagnes in data folder * cleaning code, removing unneccesary changes from main, removed normals prediction network * remove old scripts files, rename generative pipeline flag to no_input_data, revert to default histogram padding in proposal sampler * changing pipeline flag * move no input data flag from pipeline to datamanager * undo config and merge in deepfloyd from main * merging grad scaler pipeline addition * merging + setting proposal net scheduler * revert nerfacto config change * typing + linting * rename dreamfusion datamanager to random camras datamanager, check datamanager typing in pipeline and trainer * fix test import * fix test import * changing datamanager check in trainer, add typing in random cameras datamanager * add Tensor to randomcamerasdatamanager * remove old comments * resolve tyro warning * update fields to use tcnn * lint * add packages for deepfloyd * add versions * docs first draft * rename to generfacto, clean docs * fix import error * minor changes * fix orientation loss bug, retune params * cleanup * typing * trying docs fix * fixing generative package imports * fix pyright * docs fix * merge w main * docs minor change * tuning + bugfixes * remove torch.backends --------- Co-authored-by: Abhik Ahuja Co-authored-by: David McAllister Co-authored-by: Jake Austin Co-authored-by: Matthew Tancik Co-authored-by: Matt Tancik Co-authored-by: SeungWoo Yoo Co-authored-by: ayaanzhaque Co-authored-by: Ayaan Haque --- docs/nerfology/methods/generfacto.md | 39 ++ docs/nerfology/methods/index.md | 1 + nerfstudio/cameras/camera_optimizers.py | 4 +- nerfstudio/configs/experiment_config.py | 2 + nerfstudio/configs/method_configs.py | 50 ++ .../random_cameras_datamanager.py | 315 +++++++++++ nerfstudio/engine/trainer.py | 9 +- nerfstudio/field_components/field_heads.py | 1 + nerfstudio/fields/generfacto_field.py | 169 ++++++ nerfstudio/generative/deepfloyd.py | 91 +-- .../generative/positional_text_embeddings.py | 120 ++++ nerfstudio/generative/stable_diffusion.py | 37 +- nerfstudio/generative/utils.py | 35 +- .../model_components/scene_colliders.py | 2 - nerfstudio/models/base_model.py | 2 + nerfstudio/models/generfacto.py | 528 ++++++++++++++++++ nerfstudio/pipelines/base_pipeline.py | 7 +- nerfstudio/scripts/train.py | 4 + nerfstudio/utils/eval_utils.py | 1 - pyproject.toml | 3 + tests/pipelines/test_vanilla_pipeline.py | 2 +- tests/test_train.py | 1 + 22 files changed, 1324 insertions(+), 99 deletions(-) create mode 100644 docs/nerfology/methods/generfacto.md create mode 100644 nerfstudio/data/datamanagers/random_cameras_datamanager.py create mode 100644 nerfstudio/fields/generfacto_field.py create mode 100644 nerfstudio/generative/positional_text_embeddings.py create mode 100644 nerfstudio/models/generfacto.py diff --git a/docs/nerfology/methods/generfacto.md b/docs/nerfology/methods/generfacto.md new file mode 100644 index 000000000..3323b9dfd --- /dev/null +++ b/docs/nerfology/methods/generfacto.md @@ -0,0 +1,39 @@ +# Generfacto + +

Generate 3D models from text

+ +**Our model that combines generative 3D with our latest NeRF methods** + +## Installation + +First install nerfstudio dependencies. Then run: + +```bash +pip install -e .[gen] +``` + +Two options for text to image diffusion models are provided: Stable Diffusion and DeepFloyd IF. +We use Deepfloyd IF by default because it trains faster and produces better results. Using this model requires users to sign a license agreement for the model card of DeepFloyd IF, which can be found [here](https://huggingface.co/DeepFloyd/IF-I-XL-v1.0). Once the licensed is signed, log into Huggingface locally by running the following command: + +```bash +huggingface-cli login +``` + +If you do not want to sign the license agreement, you can use the Stable Diffusion model (instructions below). + +## Running Generfacto + +Once installed, run: + +```bash +ns-train generfacto --prompt "a high quality photo of a pineapple" +``` + +The first time you run this method, the diffusion model weights will be downloaded and cached +from Hugging Face, which may take a couple minutes. + +Specify which diffusion model to use with the diffusion_model flag: + +```bash +ns-train generfacto --pipeline.model.diffusion_model ["stablediffusion", "deepfloyd"] +``` diff --git a/docs/nerfology/methods/index.md b/docs/nerfology/methods/index.md index a11491c4d..8eac41800 100644 --- a/docs/nerfology/methods/index.md +++ b/docs/nerfology/methods/index.md @@ -36,6 +36,7 @@ The following methods are supported in nerfstudio: NeRFPlayer Tetra-NeRF TensoRF + Generfacto ``` (own_method_docs)= diff --git a/nerfstudio/cameras/camera_optimizers.py b/nerfstudio/cameras/camera_optimizers.py index 899833f83..0ed2473f9 100644 --- a/nerfstudio/cameras/camera_optimizers.py +++ b/nerfstudio/cameras/camera_optimizers.py @@ -53,10 +53,10 @@ class CameraOptimizerConfig(InstantiateConfig): orientation_noise_std: float = 0.0 """Noise to add to initial orientations. Useful for debugging.""" - optimizer: OptimizerConfig = AdamOptimizerConfig(lr=6e-4, eps=1e-15) + optimizer: OptimizerConfig = field(default_factory=lambda: AdamOptimizerConfig(lr=6e-4, eps=1e-15)) """ADAM parameters for camera optimization.""" - scheduler: SchedulerConfig = ExponentialDecaySchedulerConfig(max_steps=10000) + scheduler: SchedulerConfig = field(default_factory=lambda: ExponentialDecaySchedulerConfig(max_steps=10000)) """Learning rate scheduler for camera optimizer..""" param_group: tyro.conf.Suppress[str] = "camera_opt" diff --git a/nerfstudio/configs/experiment_config.py b/nerfstudio/configs/experiment_config.py index 2f57d50e4..c3a400dd1 100644 --- a/nerfstudio/configs/experiment_config.py +++ b/nerfstudio/configs/experiment_config.py @@ -72,6 +72,8 @@ class ExperimentConfig(InstantiateConfig): """Which visualizer to use.""" data: Optional[Path] = None """Alias for --pipeline.datamanager.data""" + prompt: Optional[str] = None + """Alias for --pipeline.model.prompt""" relative_model_dir: Path = Path("nerfstudio_models/") """Relative path to save all checkpoints.""" diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index e0a270ba9..b8ec7fede 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -26,7 +26,10 @@ import tyro from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig from nerfstudio.configs.base_config import ViewerConfig from nerfstudio.configs.external_methods import get_external_methods + +from nerfstudio.data.datamanagers.random_cameras_datamanager import RandomCamerasDataManagerConfig from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager, VanillaDataManagerConfig + from nerfstudio.data.dataparsers.blender_dataparser import BlenderDataParserConfig from nerfstudio.data.dataparsers.dnerf_dataparser import DNeRFDataParserConfig from nerfstudio.data.dataparsers.instant_ngp_dataparser import InstantNGPDataParserConfig @@ -47,6 +50,7 @@ from nerfstudio.engine.trainer import TrainerConfig from nerfstudio.field_components.temporal_distortions import TemporalDistortionKind from nerfstudio.fields.sdf_field import SDFFieldConfig from nerfstudio.models.depth_nerfacto import DepthNerfactoModelConfig +from nerfstudio.models.generfacto import GenerfactoModelConfig from nerfstudio.models.instant_ngp import InstantNGPModelConfig from nerfstudio.models.mipnerf import MipNerfModel from nerfstudio.models.nerfacto import NerfactoModelConfig @@ -72,6 +76,7 @@ descriptions = { "tensorf": "tensorf", "dnerf": "Dynamic-NeRF model. (slow)", "phototourism": "Uses the Phototourism data.", + "generfacto": "Generative Text to NeRF model", "neus": "Implementation of NeuS. (slow)", "neus-facto": "Implementation of NeuS-Facto. (slow)", } @@ -483,6 +488,51 @@ method_configs["phototourism"] = TrainerConfig( vis="viewer", ) +method_configs["generfacto"] = TrainerConfig( + method_name="generfacto", + experiment_name="", + steps_per_eval_batch=50, + steps_per_eval_image=50, + steps_per_save=200, + max_num_iterations=30000, + mixed_precision=True, + pipeline=VanillaPipelineConfig( + datamanager=RandomCamerasDataManagerConfig( + horizontal_rotation_warmup=3000, + ), + model=GenerfactoModelConfig( + eval_num_rays_per_chunk=1 << 15, + distortion_loss_mult=1.0, + interlevel_loss_mult=100.0, + max_res=256, + sphere_collider=True, + initialize_density=True, + taper_range=(0, 2000), + random_background=True, + proposal_warmup=2000, + proposal_update_every=0, + proposal_weights_anneal_max_num_iters=2000, + start_lambertian_training=500, + start_normals_training=2000, + opacity_loss_mult=0.001, + positional_prompting="discrete", + guidance_scale=25, + ), + ), + optimizers={ + "proposal_networks": { + "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15), + "scheduler": None, + }, + "fields": { + "optimizer": AdamOptimizerConfig(lr=5e-4, eps=1e-15), + "scheduler": None, + }, + }, + viewer=ViewerConfig(), + vis="viewer", +) + method_configs["neus"] = TrainerConfig( method_name="neus", steps_per_eval_image=500, diff --git a/nerfstudio/data/datamanagers/random_cameras_datamanager.py b/nerfstudio/data/datamanagers/random_cameras_datamanager.py new file mode 100644 index 000000000..4269b428a --- /dev/null +++ b/nerfstudio/data/datamanagers/random_cameras_datamanager.py @@ -0,0 +1,315 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Data manager without input images, only random camera poses +""" + +from __future__ import annotations + +import sys +from dataclasses import dataclass, field +from typing import Dict, List, Tuple, Type, Union + +import torch +from rich.progress import Console +from torch.nn import Parameter +from torch import Tensor +from typing_extensions import Literal + +from nerfstudio.cameras.cameras import Cameras +from nerfstudio.cameras.rays import RayBundle +from nerfstudio.configs.config_utils import to_immutable_dict +from nerfstudio.data.datamanagers.base_datamanager import DataManager, DataManagerConfig +from nerfstudio.data.datasets.base_dataset import InputDataset +from nerfstudio.data.scene_box import SceneBox +from nerfstudio.data.utils.dataloaders import RandIndicesEvalDataloader + +CONSOLE = Console(width=120) + + +class TrivialDataset(InputDataset): + """A trivial dataset with blank images for the viewer""" + + # pylint: disable=super-init-not-called + def __init__(self, cameras: Cameras): + self.size = cameras.size + self.cameras = cameras + self.alpha_color = None + self.scene_box = SceneBox(torch.Tensor([[-1, -1, -1], [1, 1, 1]])) + self.mask_filenames = None + self.metadata = to_immutable_dict({}) + + def __len__(self): + return self.size + + def __getitem__(self, index: int) -> Dict: + return { + "image": torch.cat([torch.ones(128, 256, 3), torch.zeros(128, 256, 3)], dim=0), + "image_idx": index, + } + + +def random_train_pose( + size: int, + resolution: int, + device: Union[torch.device, str], + radius_mean: float = 1.0, + radius_std: float = 0.1, + central_rotation_range: Tuple[float, float] = (0, 360), + vertical_rotation_range: Tuple[float, float] = (-90, 0), + focal_range: Tuple[float, float] = (0.75, 1.35), + jitter_std: float = 0.01, + center: Tuple[float, float, float] = (0, 0, 0), +) -> Tuple[Cameras, Tensor, Tensor]: + """generate random poses from an orbit camera + Args: + size: batch size of generated poses. + device: where to allocate the output. + radius_mean: mean radius of the orbit camera. + radius_std: standard deviation of the radius of the orbit camera. + central_rotation_range: amount that we rotate around the center of the object + vertical_rotation_range: amount that we allow the cameras to pan up and down from horizontal + focal_range: focal length range + jitter_std: standard deviation of the jitter added to the camera position + center: center of the object + Return: + poses: [size, 4, 4] + """ + + vertical_rotation_range = ( + vertical_rotation_range[0] + 90, + vertical_rotation_range[1] + 90, + ) + # This is the uniform sample on the part of the sphere we care about where 0 = 0 degrees and 1 = 360 degrees + sampled_uniform = ( + torch.rand(size) * (vertical_rotation_range[1] - vertical_rotation_range[0]) + vertical_rotation_range[0] + ) / 180 + vertical_rotation = torch.arccos(1 - 2 * sampled_uniform) + central_rotation = torch.deg2rad( + torch.rand(size) * (central_rotation_range[1] - central_rotation_range[0]) + central_rotation_range[0] + ) + + c_cos = torch.cos(central_rotation) + c_sin = torch.sin(central_rotation) + v_cos = torch.cos(vertical_rotation) + v_sin = torch.sin(vertical_rotation) + zeros = torch.zeros_like(central_rotation) + ones = torch.ones_like(central_rotation) + + rot_z = torch.stack( + [ + torch.stack([c_cos, -c_sin, zeros], dim=-1), + torch.stack([c_sin, c_cos, zeros], dim=-1), + torch.stack([zeros, zeros, ones], dim=-1), + ], + dim=-2, + ) + + rot_x = torch.stack( + [ + torch.stack([ones, zeros, zeros], dim=-1), + torch.stack([zeros, v_cos, -v_sin], dim=-1), + torch.stack([zeros, v_sin, v_cos], dim=-1), + ], + dim=-2, + ) + + # Default directions are facing in the -z direction, so origins should face opposite way + origins = torch.stack([torch.tensor([0, 0, 1])] * size, dim=0) + origins = (origins * radius_mean) + (origins * (torch.randn((origins.shape)) * radius_std)) + R = torch.bmm(rot_z, rot_x) # Want to have Rx @ Ry @ origin + t = ( + torch.bmm(R, origins.unsqueeze(-1)) + + torch.randn((size, 3, 1)) * jitter_std + + torch.tensor(center)[None, :, None] + ) + camera_to_worlds = torch.cat([R, t], dim=-1) + + focals = torch.rand(size) * (focal_range[1] - focal_range[0]) + focal_range[0] + + cameras = Cameras( + camera_to_worlds=camera_to_worlds, + fx=focals * resolution, + fy=focals * resolution, + cx=resolution / 2, + cy=resolution / 2, + ).to(device) + + return cameras, torch.rad2deg(vertical_rotation), torch.rad2deg(central_rotation) + + +@dataclass +class RandomCamerasDataManagerConfig(DataManagerConfig): + """Configuration for data manager that does not load from a dataset. Instead, it generates random poses.""" + + _target: Type = field(default_factory=lambda: RandomCamerasDataManager) + train_resolution: int = 64 + """Training resolution""" + eval_resolution: int = 64 + """Evaluation resolution""" + num_eval_angles: int = 256 + """Number of evaluation angles""" + train_images_per_batch: int = 1 + """Number of images per batch for training""" + eval_images_per_batch: int = 1 + """Number of images per batch for evaluation""" + radius_mean: float = 2.5 + """Mean radius of camera orbit""" + radius_std: float = 0.1 + """Std of radius of camera orbit""" + focal_range: Tuple[float, float] = (0.7, 1.35) + """Range of focal length""" + vertical_rotation_range: Tuple[float, float] = (-90, 0) + """Range of vertical rotation""" + jitter_std: float = 0.05 + """Std of camera direction jitter, so we don't just point the cameras towards the center every time""" + center: Tuple[float, float, float] = (0, 0, 0) + """Center coordinate of the camera sphere""" + horizontal_rotation_warmup: int = 0 + """How many steps until the full horizontal rotation range is used""" + + +class RandomCamerasDataManager(DataManager): # pylint: disable=abstract-method + """Basic stored data manager implementation. + + This is pretty much a port over from our old dataloading utilities, and is a little jank + under the hood. We may clean this up a little bit under the hood with more standard dataloading + components that can be strung together, but it can be just used as a black box for now since + only the constructor is likely to change in the future, or maybe passing in step number to the + next_train and next_eval functions. + + Args: + config: the DataManagerConfig used to instantiate class + """ + + config: RandomCamerasDataManagerConfig + + # pylint: disable=super-init-not-called + def __init__( + self, + config: RandomCamerasDataManagerConfig, + device: Union[torch.device, str] = "cpu", + test_mode: Literal["test", "val", "inference"] = "val", + world_size: int = 1, + local_rank: int = 0, + **kwargs, # pylint: disable=unused-argument + ): + self.config = config + self.device = device + self.world_size = world_size + self.local_rank = local_rank + self.sampler = None + self.test_mode = test_mode + self.test_split = "test" if test_mode in ["test", "inference"] else "val" + + if self.config.data is not None: + CONSOLE.print("[red] --data should not be used with the RandomCamerasDataManager[/red]") + sys.exit(1) + + cameras, _, _ = random_train_pose( + self.config.num_eval_angles, + self.config.eval_resolution, + device=self.device, + radius_mean=self.config.radius_mean, + radius_std=self.config.radius_std, + focal_range=self.config.focal_range, + central_rotation_range=(-180, 180), + vertical_rotation_range=self.config.vertical_rotation_range, + jitter_std=self.config.jitter_std, + center=self.config.center, + ) + + self.train_dataset = TrivialDataset(cameras) + self.eval_dataset = TrivialDataset(cameras) + + self.eval_dataloader = RandIndicesEvalDataloader( + input_dataset=self.eval_dataset, + device=self.device, + num_workers=self.world_size * 4, + ) + + # pylint: disable=non-parent-init-called + DataManager.__init__(self) + + def next_train(self, step: int) -> Tuple[RayBundle, Dict]: + """Returns the next batch of data from the train dataloader.""" + + self.train_count += 1 + horizontal_range = min((step / max(1, self.config.horizontal_rotation_warmup)), 1) * 180 + + cameras, vertical_rotation, central_rotation = random_train_pose( + self.config.train_images_per_batch, + self.config.train_resolution, + device=self.device, + radius_mean=self.config.radius_mean, + radius_std=self.config.radius_std, + focal_range=self.config.focal_range, + vertical_rotation_range=self.config.vertical_rotation_range, + jitter_std=self.config.jitter_std, + center=self.config.center, + central_rotation_range=(-horizontal_range, horizontal_range), + ) + ray_bundle = cameras.generate_rays(torch.tensor(list(range(self.config.train_images_per_batch)))).flatten() + + return ray_bundle, { + "vertical": vertical_rotation, + "central": central_rotation, + "initialization": True, + } + + def next_eval(self, step: int) -> Tuple[RayBundle, Dict]: + """Returns the next batch of data from the eval dataloader.""" + self.eval_count += 1 + + cameras, vertical_rotation, central_rotation = random_train_pose( + self.config.eval_images_per_batch, + self.config.eval_resolution, + device=self.device, + radius_mean=self.config.radius_mean, + radius_std=self.config.radius_std, + focal_range=self.config.focal_range, + vertical_rotation_range=self.config.vertical_rotation_range, + jitter_std=self.config.jitter_std, + center=self.config.center, + ) + ray_bundle = cameras.generate_rays( + torch.tensor([[i] for i in range(self.config.train_images_per_batch)]) + ).flatten() + + return ray_bundle, {"vertical": vertical_rotation, "central": central_rotation} + + def next_eval_image(self, step: int) -> Tuple[int, RayBundle, Dict]: + for camera_ray_bundle, batch in self.eval_dataloader: + assert camera_ray_bundle.camera_indices is not None + image_idx = int(camera_ray_bundle.camera_indices[0, 0, 0]) + return image_idx, camera_ray_bundle, batch + raise ValueError("No more eval images") + + def get_train_rays_per_batch(self) -> int: + return self.config.train_resolution**2 + + def get_eval_rays_per_batch(self) -> int: + return self.config.eval_resolution**2 + + def get_param_groups( + self, + ) -> Dict[str, List[Parameter]]: # pylint: disable=no-self-use + """Get the param groups for the data manager. + Returns: + A list of dictionaries containing the data manager's param groups. + """ + param_groups = {} + + return param_groups diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index afc0d94f3..4afb3a69e 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -34,6 +34,7 @@ from torch.cuda.amp.grad_scaler import GradScaler from nerfstudio.configs.experiment_config import ExperimentConfig from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes, TrainingCallbackLocation +from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager from nerfstudio.engine.optimizers import Optimizers from nerfstudio.pipelines.base_pipeline import VanillaPipeline from nerfstudio.utils import profiler, writer @@ -224,9 +225,11 @@ class Trainer: """Train the model.""" assert self.pipeline.datamanager.train_dataset is not None, "Missing DatsetInputs" - self.pipeline.datamanager.train_dataparser_outputs.save_dataparser_transform( - self.base_dir / "dataparser_transforms.json" - ) + # don't want to call save_dataparser_transform if pipeline's datamanager does not have a dataparser + if isinstance(self.pipeline.datamanager, VanillaDataManager): + self.pipeline.datamanager.train_dataparser_outputs.save_dataparser_transform( + self.base_dir / "dataparser_transforms.json" + ) self._init_viewer_state() with TimeWriter(writer, EventName.TOTAL_TRAIN_TIME): diff --git a/nerfstudio/field_components/field_heads.py b/nerfstudio/field_components/field_heads.py index ed199b4b9..44df9f602 100644 --- a/nerfstudio/field_components/field_heads.py +++ b/nerfstudio/field_components/field_heads.py @@ -34,6 +34,7 @@ class FieldHeadNames(Enum): NORMALS = "normals" PRED_NORMALS = "pred_normals" UNCERTAINTY = "uncertainty" + BACKGROUND_RGB = "background_rgb" TRANSIENT_RGB = "transient_rgb" TRANSIENT_DENSITY = "transient_density" SEMANTICS = "semantics" diff --git a/nerfstudio/fields/generfacto_field.py b/nerfstudio/fields/generfacto_field.py new file mode 100644 index 000000000..d72f5c263 --- /dev/null +++ b/nerfstudio/fields/generfacto_field.py @@ -0,0 +1,169 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Field for Generfacto model +""" + + +from typing import Optional, Tuple, Dict, Literal + +import numpy as np +import torch +from torch.nn.parameter import Parameter +from jaxtyping import Float +from torch import Tensor, nn + +from nerfstudio.field_components.mlp import MLP +from nerfstudio.cameras.rays import RayBundle, RaySamples +from nerfstudio.data.scene_box import SceneBox +from nerfstudio.field_components.activations import trunc_exp +from nerfstudio.field_components.field_heads import FieldHeadNames +from nerfstudio.field_components.encodings import HashEncoding, SHEncoding +from nerfstudio.fields.base_field import Field + + +def get_normalized_directions(directions: Float[Tensor, "bs 3"]): + """SH encoding must be in the range [0, 1] + + Args: + directions: batch of directions + """ + return (directions + 1.0) / 2.0 + + +class GenerfactoField(Field): + """Generfacto Field that uses TCNN + + Args: + aabb: parameters of scene aabb bounds + num_layers: number of hidden layers + hidden_dim: dimension of hidden layers + geo_feat_dim: output geo feat dimensions + num_levels: number of levels of the hashmap for the base mlp + max_res: maximum resolution of the hashmap for the base mlp + log2_hashmap_size: size of the hashmap for the base mlp + num_layers_color: number of hidden layers for color network + hidden_dim_color: dimension of hidden layers for color network + """ + + def __init__( + self, + aabb: Tensor, + num_layers: int = 2, + hidden_dim: int = 64, + geo_feat_dim: int = 15, + num_levels: int = 16, + max_res: int = 256, + log2_hashmap_size: int = 19, + num_layers_color: int = 3, + hidden_dim_color: int = 64, + implementation: Literal["tcnn", "torch"] = "tcnn", + ) -> None: + super().__init__() + + self.aabb = Parameter(aabb, requires_grad=False) + self.geo_feat_dim = geo_feat_dim + + base_res = 16 + features_per_level = 2 + np.exp((np.log(max_res) - np.log(base_res)) / (num_levels - 1)) + + self.direction_encoding = SHEncoding( + levels=4, + implementation=implementation, + ) + + encoder = HashEncoding( + num_levels=num_levels, + min_res=base_res, + max_res=max_res, + log2_hashmap_size=log2_hashmap_size, + features_per_level=features_per_level, + implementation=implementation, + ) + network = MLP( + in_dim=encoder.get_out_dim(), + num_layers=num_layers, + layer_width=hidden_dim, + out_dim=1 + self.geo_feat_dim, + activation=nn.ReLU(), + out_activation=None, + implementation=implementation, + ) + self.mlp_base = torch.nn.Sequential(encoder, network) + + self.mlp_background_color = MLP( + in_dim=self.direction_encoding.get_out_dim(), + num_layers=2, + layer_width=32, + out_dim=3, + activation=nn.ReLU(), + out_activation=nn.Sigmoid(), + implementation=implementation, + ) + + self.mlp_head = MLP( + in_dim=self.geo_feat_dim, + num_layers=num_layers_color, + layer_width=hidden_dim_color, + out_dim=3, + activation=nn.ReLU(), + out_activation=nn.Sigmoid(), + implementation=implementation, + ) + + def get_density(self, ray_samples: RaySamples) -> Tuple[Tensor, Tensor]: + """Computes and returns the densities.""" + positions = SceneBox.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) + self._sample_locations = positions + if not self._sample_locations.requires_grad: + self._sample_locations.requires_grad = True + positions_flat = positions.view(-1, 3) + h = self.mlp_base(positions_flat).view(*ray_samples.frustums.shape, -1) + density_before_activation, base_mlp_out = torch.split(h, [1, self.geo_feat_dim], dim=-1) + self._density_before_activation = density_before_activation + + # Rectifying the density with an exponential is much more stable than a ReLU or + # softplus, because it enables high post-activation (float32) density outputs + # from smaller internal (float16) parameters. + density = trunc_exp(density_before_activation.to(positions)) + return density, base_mlp_out + + def get_background_rgb(self, ray_bundle: RayBundle) -> Tensor: + """Predicts background colors at infinity.""" + directions = get_normalized_directions(ray_bundle.directions) + + outputs_shape = ray_bundle.directions.shape[:-1] + directions_flat = self.direction_encoding(directions.view(-1, 3)) + background_rgb = self.mlp_background_color(directions_flat).view(*outputs_shape, -1).to(directions) + + return background_rgb + + def get_outputs( + self, ray_samples: RaySamples, density_embedding: Optional[Tensor] = None + ) -> Dict[FieldHeadNames, Tensor]: + assert density_embedding is not None + outputs = {} + + directions = get_normalized_directions(ray_samples.frustums.directions) + + outputs_shape = ray_samples.frustums.directions.shape[:-1] + + h = density_embedding.view(-1, self.geo_feat_dim) + + rgb = self.mlp_head(h).view(*outputs_shape, -1).to(directions) + outputs.update({FieldHeadNames.RGB: rgb}) + + return outputs diff --git a/nerfstudio/generative/deepfloyd.py b/nerfstudio/generative/deepfloyd.py index 982be0879..72c6cc373 100644 --- a/nerfstudio/generative/deepfloyd.py +++ b/nerfstudio/generative/deepfloyd.py @@ -14,21 +14,30 @@ import gc from pathlib import Path -from typing import List, Optional, Union, cast +from typing import List, Optional, Union import torch import torch.nn.functional as F import tyro -from diffusers import IFPipeline -from diffusers.pipelines.deepfloyd_if import IFPipelineOutput + from jaxtyping import Float from PIL import Image -from torch import FloatTensor, Generator, Tensor, nn +from torch import Generator, Tensor, nn from torch.cuda.amp.grad_scaler import GradScaler -from transformers import T5EncoderModel -from nerfstudio.generative.utils import _SDSGradient +from nerfstudio.generative.utils import CatchMissingPackages + +try: + from diffusers import IFPipeline as IFOrig + from diffusers.pipelines.deepfloyd_if import IFPipelineOutput as IFOutputOrig + + from diffusers import IFPipeline, DiffusionPipeline + from diffusers.pipelines.deepfloyd_if import IFPipelineOutput + from transformers import T5EncoderModel + +except ImportError: + IFPipeline = IFPipelineOutput = T5EncoderModel = CatchMissingPackages() IMG_DIM = 64 @@ -61,7 +70,7 @@ class DeepFloyd(nn.Module): variant="fp16", torch_dtype=torch.float16, ) - assert isinstance(self.pipe, IFPipeline) + assert isinstance(self.pipe, DiffusionPipeline) self.pipe = self.pipe.to(self.device) self.pipe.enable_attention_slicing(1) @@ -96,7 +105,7 @@ class DeepFloyd(nn.Module): variant="fp16", torch_dtype=torch.float16, ) - assert isinstance(self.pipe, IFPipeline) + assert isinstance(self.pipe, DiffusionPipeline) self.pipe = self.pipe.to(self.device) self.pipe.enable_attention_slicing(1) @@ -122,12 +131,12 @@ class DeepFloyd(nn.Module): prompt = [prompt] if isinstance(prompt, str) else prompt negative_prompt = [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt - assert isinstance(self.pipe, IFPipeline) + assert isinstance(self.pipe, DiffusionPipeline) with torch.no_grad(): prompt_embeds, negative_embeds = self.pipe.encode_prompt(prompt, negative_prompt=negative_prompt) - assert isinstance(negative_embeds, FloatTensor) - assert isinstance(prompt_embeds, FloatTensor) + assert isinstance(negative_embeds, Tensor) + assert isinstance(prompt_embeds, Tensor) return torch.cat([negative_embeds, prompt_embeds]) def sds_loss( @@ -146,40 +155,37 @@ class DeepFloyd(nn.Module): Returns: The loss """ - with torch.autocast(device_type="cuda", enabled=False): - image = F.interpolate(image.half(), (IMG_DIM, IMG_DIM), mode="bilinear", align_corners=False) - t = torch.randint(self.min_step, self.max_step + 1, [1], dtype=torch.long, device=self.device) + image = F.interpolate(image.half(), (IMG_DIM, IMG_DIM), mode="bilinear", align_corners=False) + t = torch.randint(self.min_step, self.max_step + 1, [1], dtype=torch.long, device=self.device) - # predict the noise residual with unet, NO grad! - with torch.no_grad(): - # add noise - noise = torch.randn_like(image) - image_noisy = self.scheduler.add_noise(image, noise, t) # type: ignore - # pred noise - image_model_input = torch.cat((image_noisy,) * 2) - noise_pred = self.unet(image_model_input, t, encoder_hidden_states=text_embeddings).sample + # predict the noise residual with unet, NO grad! + with torch.no_grad(): + # add noise + noise = torch.randn_like(image) + image_noisy = self.scheduler.add_noise(image, noise, t) # type: ignore + # pred noise + image_model_input = torch.cat((image_noisy,) * 2) + noise_pred = self.unet(image_model_input, t, encoder_hidden_states=text_embeddings).sample - # perform guidance - noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) - noise_pred_text, predicted_variance = noise_pred_text.split(3, dim=1) - noise_pred_uncond, _ = noise_pred_uncond.split(3, dim=1) + # perform guidance + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred_text, predicted_variance = noise_pred_text.split(3, dim=1) + noise_pred_uncond, _ = noise_pred_uncond.split(3, dim=1) - noise_pred = noise_pred_text + guidance_scale * (noise_pred_text - noise_pred_uncond) + noise_pred = noise_pred_text + guidance_scale * (noise_pred_text - noise_pred_uncond) - # w(t), sigma_t^2 - w = 1 - self.alphas[t] + # w(t), sigma_t^2 + w = 1 - self.alphas[t] - grad = w * (noise_pred - noise) - grad = torch.nan_to_num(grad) + grad = w * (noise_pred - noise) + grad = torch.nan_to_num(grad) - if grad_scaler is not None: - loss = cast(Tensor, _SDSGradient.apply(grad_scaler.scale(image), grad)) - else: - loss = cast(Tensor, _SDSGradient.apply(image, grad)) + target = (image - grad).detach() + loss = 0.5 * F.mse_loss(image, target, reduction="sum") / image.shape[0] return loss - def prompt_to_img( + def prompt_to_image( self, prompts: Union[str, List[str]], negative_prompts: Union[str, List[str]] = "", @@ -201,16 +207,17 @@ class DeepFloyd(nn.Module): prompts = [prompts] if isinstance(prompts, str) else prompts negative_prompts = [negative_prompts] if isinstance(negative_prompts, str) else negative_prompts - - assert isinstance(self.pipe, IFPipeline) + assert isinstance(self.pipe, DiffusionPipeline) prompt_embeds, negative_embeds = self.pipe.encode_prompt(prompts, negative_prompt=negative_prompts) + + assert isinstance(self.pipe, IFOrig) model_output = self.pipe( prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_embeds, generator=generator ) - assert isinstance(model_output, IFPipelineOutput) - image = model_output.images[0] + assert isinstance(model_output, IFOutputOrig) + output_image = model_output.images[0] - return image + return output_image def generate_image( @@ -228,7 +235,7 @@ def generate_image( cuda_device = torch.device("cuda") with torch.no_grad(): df = DeepFloyd(cuda_device) - img = df.prompt_to_img(prompt, negative, generator, steps) + img = df.prompt_to_image(prompt, negative, generator, steps) img.save(save_path) diff --git a/nerfstudio/generative/positional_text_embeddings.py b/nerfstudio/generative/positional_text_embeddings.py new file mode 100644 index 000000000..4ba62b024 --- /dev/null +++ b/nerfstudio/generative/positional_text_embeddings.py @@ -0,0 +1,120 @@ +# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utility helper functions for generative 3D models""" + +import torch +from torch import Tensor +from jaxtyping import Float +from typing_extensions import Literal +from typing import Union + +from nerfstudio.generative.stable_diffusion import StableDiffusion +from nerfstudio.generative.deepfloyd import DeepFloyd + + +class PositionalTextEmbeddings: + """Postional Prompts. Currently the following location based text embeddings are supported: + - discrete: Choose the embedding closest to the camera position + - interpolated: Interpolate between the embeddings based on the camera position + - off: Don't modify the text embedding based on the camera position + + Args: + base_prompt: Prompt for base view + top_prompt: Prompt for top view + side_prompt: Prompt for side view + back_prompt: Prompt for back view + front_prompt: Prompt for front view + diffusion_model: Instance of StableDiffusion + positional_prompting: how to incorporate position into prompt. + """ + + def __init__( + self, + base_prompt: str, + top_prompt: str, + side_prompt: str, + back_prompt: str, + front_prompt: str, + diffusion_model: Union[StableDiffusion, DeepFloyd], + positional_prompting: Literal["discrete", "interpolated", "off"] = "discrete", + ): + self.positional_prompting = positional_prompting + self.diffusion_device = diffusion_model.device + self.diffusion = diffusion_model + self.update_prompt(base_prompt, top_prompt, side_prompt, back_prompt, front_prompt) + + def update_prompt(self, base_prompt: str, top_prompt: str, side_prompt: str, back_prompt: str, front_prompt: str): + """Update the text embeddings based on the new prompts. + + Args: + base_prompt: Prompt for base view + top_prompt: Prompt for top view + side_prompt: Prompt for side view + back_prompt: Prompt for back view + front_prompt: Prompt for front view + """ + self.base_embed = self.diffusion.get_text_embeds(base_prompt, "") + self.top_embed = self.diffusion.get_text_embeds(top_prompt, "") + self.side_embed = self.diffusion.get_text_embeds(side_prompt, "") + self.back_embed = self.diffusion.get_text_embeds(back_prompt, "") + self.front_embed = self.diffusion.get_text_embeds(front_prompt, "") + + if isinstance(self.diffusion, DeepFloyd): + self.diffusion.delete_text_encoder() + + def get_text_embedding( + self, vertical_angle: Float[Tensor, "1"], horizontal_angle: Float[Tensor, "1"] + ) -> Float[Tensor, "2 max_length embed_dim"]: + """Get text embedding based on the position of the camera relative to the scene. + This trick is used in Dreamfusion (https://dreamfusion3d.github.io/). + + Args: + vertical_angle: vertical angle of the camera + horizonal_angle: horizonal angle of the camera + """ + # set horizontal_angle between 0, 360 + horizontal_angle = torch.fmod(horizontal_angle, 360) + horizontal_angle = torch.where(horizontal_angle < 0, horizontal_angle + 360, horizontal_angle) + + if self.positional_prompting == "discrete": + if vertical_angle < 40: + text_embedding = self.top_embed + elif 315 < horizontal_angle or horizontal_angle <= 45: + text_embedding = self.front_embed + elif 45 < horizontal_angle <= 135: + text_embedding = self.side_embed + elif 135 < horizontal_angle <= 225: + text_embedding = self.back_embed + else: # horizontal_angle > 225 and horizontal_angle <= 315: + text_embedding = self.side_embed + elif self.positional_prompting == "interpolated": + horiz = horizontal_angle.detach().numpy()[0] + vert = max(vertical_angle.detach().numpy()[0], 0) + + if 0 < horizontal_angle <= 90: + text_embedding = (horiz) * self.side_embed + (90 - horiz) * self.front_embed + elif 90 < horizontal_angle <= 180: + text_embedding = (horiz - 90) * self.back_embed + (180 - horiz) * self.side_embed + elif 180 < horizontal_angle <= 270: + text_embedding = (horiz - 180) * self.side_embed + (270 - horiz) * self.back_embed + else: # 270 < horizontal_angle <= 360: + text_embedding = (horiz - 270) * self.front_embed + (360 - horiz) * self.side_embed + + text_embedding = text_embedding / 90.0 + text_embedding = (vert * text_embedding + (90 - vert) * self.top_embed) / 90.0 + else: + text_embedding = self.base_embed + + return text_embedding diff --git a/nerfstudio/generative/stable_diffusion.py b/nerfstudio/generative/stable_diffusion.py index 6202a9046..7e7f3dac4 100644 --- a/nerfstudio/generative/stable_diffusion.py +++ b/nerfstudio/generative/stable_diffusion.py @@ -16,10 +16,9 @@ # Modified from https://github.com/ashawkey/stable-dreamfusion/blob/main/nerf/sd.py -import sys -from dataclasses import dataclass from pathlib import Path -from typing import List, Optional, Union, cast +from typing import List, Optional, Union +from nerfstudio.utils.rich_utils import CONSOLE import mediapy import numpy as np @@ -30,22 +29,16 @@ from jaxtyping import Float from torch import Tensor, nn from torch.cuda.amp.grad_scaler import GradScaler -from nerfstudio.utils.rich_utils import CONSOLE - -from nerfstudio.generative.utils import _SDSGradient +from nerfstudio.generative.utils import CatchMissingPackages try: - from diffusers import PNDMScheduler, StableDiffusionPipeline - from transformers import logging + from diffusers import PNDMScheduler, StableDiffusionPipeline, DiffusionPipeline except ImportError: - CONSOLE.print("[bold red]Missing Stable Diffusion packages.") - CONSOLE.print(r"Install using [yellow]pip install nerfstudio\[gen][/yellow]") - CONSOLE.print(r"or [yellow]pip install -e .\[gen][/yellow] if installing from source.") - sys.exit(1) + PNDMScheduler = StableDiffusionPipeline = CatchMissingPackages() + -logging.set_verbosity_error() IMG_DIM = 512 CONST_SCALE = 0.18215 SD_IDENTIFIERS = { @@ -55,13 +48,6 @@ SD_IDENTIFIERS = { } -@dataclass -class UNet2DConditionOutput: - """Class to hold traced model""" - - sample: torch.FloatTensor - - class StableDiffusion(nn.Module): """Stable Diffusion implementation Args: @@ -88,7 +74,8 @@ class StableDiffusion(nn.Module): sd_id = SD_IDENTIFIERS[version] pipe = StableDiffusionPipeline.from_pretrained(sd_id, torch_dtype=torch.float16) - assert isinstance(pipe, StableDiffusionPipeline) + + assert isinstance(pipe, DiffusionPipeline) # and hasattr(pipe, "to") pipe = pipe.to(self.device) pipe.enable_attention_slicing() @@ -154,7 +141,7 @@ class StableDiffusion(nn.Module): Returns: The loss """ - image = F.interpolate(image, (IMG_DIM, IMG_DIM), mode="bilinear") + image = F.interpolate(image, (IMG_DIM, IMG_DIM), mode="bilinear").to(torch.float16) t = torch.randint(self.min_step, self.max_step + 1, [1], dtype=torch.long, device=self.device) latents = self.imgs_to_latent(image) @@ -177,9 +164,8 @@ class StableDiffusion(nn.Module): grad = w * (noise_pred - noise) grad = torch.nan_to_num(grad) - if grad_scaler is not None: - latents = grad_scaler.scale(latents) - loss = cast(Tensor, _SDSGradient.apply(latents, grad)) + target = (latents - grad).detach() + loss = 0.5 * F.mse_loss(latents, target, reduction="sum") / latents.shape[0] return loss @@ -230,6 +216,7 @@ class StableDiffusion(nn.Module): # compute the previous noisy sample x_t -> x_t-1 latents = self.scheduler.step(noise_pred, t, latents)["prev_sample"] # type: ignore + assert isinstance(latents, Tensor) return latents def latents_to_img(self, latents: Float[Tensor, "BS 4 H W"]) -> Float[Tensor, "BS 3 H W"]: diff --git a/nerfstudio/generative/utils.py b/nerfstudio/generative/utils.py index e5996da6d..ec5a72844 100644 --- a/nerfstudio/generative/utils.py +++ b/nerfstudio/generative/utils.py @@ -12,28 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Utility helper functions for diffusion models""" -import torch -from torch.cuda.amp import custom_bwd, custom_fwd - -"""Gradient helper functions for SDS loss""" +import sys +from nerfstudio.utils.rich_utils import CONSOLE -class _SDSGradient(torch.autograd.Function): - """Custom gradient function for SDS loss. Since it is already computed, we can just return it.""" +class CatchMissingPackages: + """Class to catch missing environment packages related to diffusion models.""" - @staticmethod - @custom_fwd - def forward(ctx, input_tensor, gt_grad): - del input_tensor - ctx.save_for_backward(gt_grad) - # Return magniture of gradient, not the actual loss. - return torch.mean(gt_grad**2) ** 0.5 + def __init__(self): + pass - @staticmethod - @custom_bwd - def backward(ctx, grad): - del grad - (gt_grad,) = ctx.saved_tensors - batch_size = len(gt_grad) - return gt_grad / batch_size, None + def __call__(self, *args, **kwargs): + CONSOLE.print("[bold red]Missing Stable Diffusion packages.") + CONSOLE.print(r"Install using [yellow]pip install nerfstudio\[gen][/yellow]") + CONSOLE.print(r"or [yellow]pip install -e .\[gen][/yellow] if installing from source.") + sys.exit(1) + + def __getattr__(self, attr): + return self.__call__ diff --git a/nerfstudio/model_components/scene_colliders.py b/nerfstudio/model_components/scene_colliders.py index 3e2e793ae..69c073b3b 100644 --- a/nerfstudio/model_components/scene_colliders.py +++ b/nerfstudio/model_components/scene_colliders.py @@ -24,7 +24,6 @@ from torch import Tensor, nn from nerfstudio.cameras.rays import RayBundle from nerfstudio.data.scene_box import SceneBox -from nerfstudio.utils.misc import torch_compile class SceneCollider(nn.Module): @@ -109,7 +108,6 @@ class AABBBoxCollider(SceneCollider): return ray_bundle -@torch_compile(dynamic=True, mode="reduce-overhead") def _intersect_with_sphere( rays_o: torch.Tensor, rays_d: torch.Tensor, center: torch.Tensor, radius: float = 1.0, near_plane: float = 0.0 ): diff --git a/nerfstudio/models/base_model.py b/nerfstudio/models/base_model.py index 1feb22adb..6f31c3b30 100644 --- a/nerfstudio/models/base_model.py +++ b/nerfstudio/models/base_model.py @@ -50,6 +50,8 @@ class ModelConfig(InstantiateConfig): """parameters to instantiate density field with""" eval_num_rays_per_chunk: int = 4096 """specifies number of rays per chunk during eval""" + prompt: Optional[str] = None + """A prompt to be used in text to NeRF models""" class Model(nn.Module): diff --git a/nerfstudio/models/generfacto.py b/nerfstudio/models/generfacto.py new file mode 100644 index 000000000..bf34eb335 --- /dev/null +++ b/nerfstudio/models/generfacto.py @@ -0,0 +1,528 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Nerfstudio's Text to 3D model. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Tuple, Type + +import numpy as np +import torch +from torch.nn import Parameter +from typing_extensions import Literal + +from nerfstudio.cameras.rays import RayBundle +from nerfstudio.engine.callbacks import ( + TrainingCallback, + TrainingCallbackAttributes, + TrainingCallbackLocation, +) +from nerfstudio.field_components.field_heads import FieldHeadNames +from nerfstudio.fields.density_fields import HashMLPDensityField +from nerfstudio.fields.generfacto_field import GenerfactoField +from nerfstudio.generative.stable_diffusion import StableDiffusion +from nerfstudio.generative.deepfloyd import DeepFloyd +from nerfstudio.generative.positional_text_embeddings import PositionalTextEmbeddings +from nerfstudio.model_components.losses import ( + MSELoss, + distortion_loss, + interlevel_loss, + orientation_loss, +) +from nerfstudio.model_components.ray_samplers import ( + ProposalNetworkSampler, + UniformSampler, +) +from nerfstudio.model_components.renderers import ( + AccumulationRenderer, + DepthRenderer, + NormalsRenderer, + RGBRenderer, +) +from nerfstudio.model_components.scene_colliders import AABBBoxCollider, SphereCollider +from nerfstudio.model_components.shaders import LambertianShader, NormalsShader +from nerfstudio.models.base_model import Model, ModelConfig +from nerfstudio.utils import colormaps, colors, math, misc + + +@dataclass +class GenerfactoModelConfig(ModelConfig): + """Generfacto model config""" + + _target: Type = field(default_factory=lambda: GenerfactoModel) + """target class to instantiate""" + prompt: str = "a high quality photo of a ripe pineapple" + """prompt for stable dreamfusion""" + + orientation_loss_mult: Tuple[float, float] = (0.001, 10.0) + """Orientation loss multipier on computed normals.""" + orientation_loss_mult_range: Tuple[int, int] = (0, 15000) + """number of iterations to reach last orientation_loss_mult value""" + random_light_source: bool = True + """Randomizes light source per output.""" + initialize_density: bool = True + """Initialize density in center of scene.""" + taper_range: Tuple[int, int] = (0, 1000) + """Range of step values for the density tapering""" + taper_strength: Tuple[float, float] = (1.0, 0.0) + """Strength schedule of center density""" + sphere_collider: bool = True + """Use spherical collider instead of box""" + random_background: bool = True + """Randomly choose between using background mlp and random color for background""" + + target_transmittance_start: float = 0.4 + """target transmittance for opacity penalty. This is the percent of the scene that is + background when rendered at the start of training""" + target_transmittance_end: float = 0.7 + """target transmittance for opacity penalty. This is the percent of the scene that is + background when rendered at the end of training""" + transmittance_end_schedule: int = 1500 + """number of iterations to reach target_transmittance_end""" + + num_proposal_samples_per_ray: Tuple[int, ...] = (256, 96) + """Number of samples per ray for each proposal network.""" + num_nerf_samples_per_ray: int = 48 + """Number of samples per ray for the nerf network.""" + proposal_update_every: int = 5 + """Sample every n steps after the warmup""" + proposal_warmup: int = 5000 + """Scales n from 1 to proposal_update_every over this many steps""" + num_proposal_iterations: int = 2 + """Number of proposal network iterations.""" + use_same_proposal_network: bool = False + """Use the same proposal network. Otherwise use different ones.""" + proposal_net_args_list: List[Dict] = field( + default_factory=lambda: [ + {"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 128}, + {"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 256}, + ] + ) + """Arguments for the proposal density fields.""" + proposal_weights_anneal_slope: float = 10.0 + """Slope of the annealing function for the proposal weights.""" + proposal_weights_anneal_max_num_iters: int = 500 + """Max num iterations for the annealing function.""" + use_single_jitter: bool = True + """Whether use single jitter or not for the proposal networks.""" + interlevel_loss_mult: float = 1.0 + """Proposal loss multiplier.""" + distortion_loss_mult: float = 1.0 + """Distortion loss multiplier.""" + start_normals_training: int = 1000 + """Start training normals after this many iterations""" + start_lambertian_training: int = 1000 + """start training with lambertian shading after this many iterations""" + opacity_penalty: bool = True + """enables penalty to encourage sparse weights (penalizing for uniform density along ray)""" + opacity_loss_mult: float = 1 + """scale for opacity penalty""" + max_res: int = 256 + """Maximum resolution of the density field.""" + + location_based_prompting: bool = True + """enables location based prompting""" + interpolated_prompting: bool = False + """enables interpolated location prompting""" + positional_prompting: Literal["discrete", "interpolated", "off"] = "discrete" + """ how to incorporate position into prompt""" + top_prompt: str = ", overhead view" + """appended to prompt for overhead view""" + side_prompt: str = ", side view" + """appended to prompt for side view""" + front_prompt: str = ", front view" + """appended to prompt for front view""" + back_prompt: str = ", back view" + """appended to prompt for back view""" + guidance_scale: float = 20 + """guidance scale for sds loss""" + diffusion_device: Optional[str] = None + """device for diffusion model""" + diffusion_model: Literal["stablediffusion", "deepfloyd"] = "deepfloyd" + """diffusion model for SDS loss""" + sd_version: str = "1-5" + """model version when using stable diffusion""" + implementation: Literal["tcnn", "torch"] = "tcnn" + """Which implementation to use for the model.""" + + +class GenerfactoModel(Model): + """Generfacto Model + + Args: + config: Generfacto configuration to instantiate model + """ + + config: GenerfactoModelConfig + + def __init__( + self, + config: GenerfactoModelConfig, + **kwargs, + ) -> None: + self.prompt = config.prompt + self.cur_prompt = config.prompt + self.sd_version = config.sd_version + self.initialize_density = config.initialize_density + self.train_normals = False + self.train_shaded = False + self.random_background = config.random_background + self.density_strength = 1.0 + self.target_transmittance = config.target_transmittance_start + self.grad_scaler = kwargs["grad_scaler"] + + self.guidance_scale = config.guidance_scale + self.top_prompt = config.top_prompt + self.side_prompt = config.side_prompt + self.back_prompt = config.back_prompt + self.front_prompt = config.front_prompt + + self.diffusion_device = ( + torch.device(kwargs["device"]) if config.diffusion_device is None else torch.device(config.diffusion_device) + ) + + super().__init__(config=config, **kwargs) + + def populate_modules(self): + """Set the fields and modules""" + super().populate_modules() + + if self.config.diffusion_model == "stablediffusion": + self._diffusion_model = StableDiffusion(self.diffusion_device, version=self.sd_version) + elif self.config.diffusion_model == "deepfloyd": + self._diffusion_model = DeepFloyd(self.diffusion_device) + + self.text_embeddings = PositionalTextEmbeddings( + base_prompt=self.cur_prompt, + top_prompt=self.cur_prompt + self.top_prompt, + side_prompt=self.cur_prompt + self.side_prompt, + back_prompt=self.cur_prompt + self.back_prompt, + front_prompt=self.cur_prompt + self.front_prompt, + diffusion_model=self._diffusion_model, + positional_prompting=self.config.positional_prompting, + ) + + # setting up fields + self.field = GenerfactoField(self.scene_box.aabb, max_res=self.config.max_res) + + # samplers + self.density_fns = [] + num_prop_nets = self.config.num_proposal_iterations + # Build the proposal network(s) + self.proposal_networks = torch.nn.ModuleList() + + for i in range(num_prop_nets): + prop_net_args = self.config.proposal_net_args_list[min(i, len(self.config.proposal_net_args_list) - 1)] + network = HashMLPDensityField( + self.scene_box.aabb, **prop_net_args, implementation=self.config.implementation + ) + self.proposal_networks.append(network) + self.density_fns.extend([network.density_fn for network in self.proposal_networks]) + + def update_schedule(step): + return np.clip( + np.interp(step, [0, self.config.proposal_warmup], [0, self.config.proposal_update_every]), + 1, + self.config.proposal_update_every, + ) + + self.proposal_sampler = ProposalNetworkSampler( + num_nerf_samples_per_ray=self.config.num_nerf_samples_per_ray, + num_proposal_samples_per_ray=self.config.num_proposal_samples_per_ray, + num_proposal_network_iterations=self.config.num_proposal_iterations, + single_jitter=self.config.use_single_jitter, + update_sched=update_schedule, + initial_sampler=UniformSampler(single_jitter=self.config.use_single_jitter), + ) + + # renderers + self.renderer_rgb = RGBRenderer(background_color=colors.WHITE) + self.renderer_accumulation = AccumulationRenderer() + self.renderer_depth = DepthRenderer() + self.renderer_normals = NormalsRenderer() + + # shaders + self.shader_lambertian = LambertianShader() + self.shader_normals = NormalsShader() + + # losses + self.rgb_loss = MSELoss() + + # colliders + if self.config.sphere_collider: + self.collider = SphereCollider(torch.Tensor([0, 0, 0]), 1.0) + else: + self.collider = AABBBoxCollider(scene_box=self.scene_box) + + def get_training_callbacks( + self, training_callback_attributes: TrainingCallbackAttributes + ) -> List[TrainingCallback]: + # the callback that we want to run every X iterations after the training iteration + def taper_density( + self, training_callback_attributes: TrainingCallbackAttributes, step: int # pylint: disable=unused-argument + ): + self.density_strength = np.interp(step, self.config.taper_range, self.config.taper_strength) + + def start_training_normals( + self, training_callback_attributes: TrainingCallbackAttributes, step: int # pylint: disable=unused-argument + ): + self.train_normals = True + + def start_shaded_training( + self, training_callback_attributes: TrainingCallbackAttributes, step: int # pylint: disable=unused-argument + ): + self.train_shaded = True + + def update_orientation_loss_mult( + self, training_callback_attributes: TrainingCallbackAttributes, step: int # pylint: disable=unused-argument + ): + if step <= self.config.start_normals_training: + self.orientation_loss_mult = 0 + else: + self.orientation_loss_mult = np.interp( + step, + self.config.orientation_loss_mult_range, + self.config.orientation_loss_mult, + ) + + # anneal the weights of the proposal network before doing PDF sampling + def set_anneal(step): + # https://arxiv.org/pdf/2111.12077.pdf eq. 18 + train_frac = np.clip(step / self.config.proposal_weights_anneal_max_num_iters, 0, 1) + + def bias(x, b): + return b * x / ((b - 1) * x + 1) + + anneal = bias(train_frac, self.config.proposal_weights_anneal_slope) + self.proposal_sampler.set_anneal(anneal) + + callbacks = [ + TrainingCallback( + where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION], + func=taper_density, + update_every_num_iters=1, + args=[self, training_callback_attributes], + ), + TrainingCallback( + where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION], + iters=(self.config.start_normals_training,), + func=start_training_normals, + args=[self, training_callback_attributes], + ), + TrainingCallback( + where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION], + iters=(self.config.start_lambertian_training,), + func=start_shaded_training, + args=[self, training_callback_attributes], + ), + TrainingCallback( + where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION], + update_every_num_iters=1, + func=update_orientation_loss_mult, + args=[self, training_callback_attributes], + ), + TrainingCallback( + where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION], + update_every_num_iters=1, + func=self.proposal_sampler.step_cb, + ), + TrainingCallback( + where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION], + update_every_num_iters=1, + func=set_anneal, + ), + ] + return callbacks + + def get_param_groups(self) -> Dict[str, List[Parameter]]: + param_groups = {} + param_groups["proposal_networks"] = list(self.proposal_networks.parameters()) + param_groups["fields"] = list(self.field.parameters()) + return param_groups + + def get_outputs(self, ray_bundle: RayBundle): # pylint: disable=too-many-statements + # uniform sampling + background_rgb = self.field.get_background_rgb(ray_bundle) + ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns) + field_outputs = self.field(ray_samples, compute_normals=True) + density = field_outputs[FieldHeadNames.DENSITY] + + if self.initialize_density: + pos = ray_samples.frustums.get_positions() + density_blob = self.density_strength * (-torch.exp(torch.norm(pos, dim=-1) / 0.4) + 2)[..., None] + density = torch.max(density + density_blob, torch.tensor([0.0], device=self.device)) + + weights = ray_samples.get_weights(density) + weights_list.append(weights) + ray_samples_list.append(ray_samples) + + accumulation = self.renderer_accumulation(weights) + depth = self.renderer_depth(weights, ray_samples) + rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights) + + accum_mask = torch.clamp((torch.nan_to_num(accumulation, nan=0.0)), min=0.0, max=1.0) + accum_mask_inv = 1.0 - accum_mask + + background = accum_mask_inv * background_rgb + + outputs = { + "rgb_only": rgb, + "background_rgb": background_rgb, + "background": background, + "accumulation": accum_mask, + "depth": depth, + } + + # These use a lot of GPU memory, so we avoid storing them for eval. + if self.training: + outputs["weights_list"] = weights_list + outputs["ray_samples_list"] = ray_samples_list + + for i in range(self.config.num_proposal_iterations): + outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i]) + + normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights) + + # lambertian shading + if self.config.random_light_source: # and self.training: + light_d = ray_bundle.origins[0] + torch.randn(3, dtype=torch.float).to(normals) + else: + light_d = ray_bundle.origins[0] + light_d = math.safe_normalize(light_d) + + if (self.train_shaded and np.random.random_sample() > 0.75) or not self.training: + shading_weight = 0.9 + else: + shading_weight = 0.0 + + shaded, shaded_albedo = self.shader_lambertian( + rgb=rgb, normals=normals, light_direction=light_d, shading_weight=shading_weight, detach_normals=False + ) + + shaded, shaded_albedo = accum_mask * shaded, accum_mask * shaded_albedo + + outputs["normals"] = self.shader_normals(normals, weights=accum_mask) + outputs["shaded"] = shaded + outputs["other_train_output"] = shaded_albedo + background + outputs["shaded_albedo"] = shaded_albedo + outputs["rgb"] = accum_mask * rgb + background + + # while training 50% of the time use a random background + if np.random.random_sample() < 0.5 and self.random_background and self.training: + background = torch.ones_like(background) * torch.rand(3, device=self.device) * accum_mask_inv + + if shading_weight > 0: + samp = np.random.random_sample() + if samp > 0.5: + outputs["train_output"] = outputs["shaded"] + else: + outputs["train_output"] = shaded_albedo + background + else: + outputs["train_output"] = accum_mask * rgb + background + + outputs["rendered_orientation_loss"] = orientation_loss( + weights.detach(), field_outputs[FieldHeadNames.NORMALS], ray_bundle.directions + ) + + assert weights.shape[-1] == 1 + if self.config.opacity_penalty: + outputs["opacity_loss"] = torch.sqrt(torch.sum(weights, dim=-2) ** 2 + 0.01) * self.config.opacity_loss_mult + + return outputs + + def get_loss_dict(self, outputs, batch, metrics_dict=None) -> Dict[str, torch.Tensor]: + # Scaling metrics by coefficients to create the losses. + + loss_dict = {} + loss_dict = misc.scale_dict(loss_dict, self.config.loss_coefficients) + if self.train_normals: + # orientation loss for computed normals + loss_dict["orientation_loss"] = self.orientation_loss_mult * torch.mean( + outputs["rendered_orientation_loss"] + ) + else: + loss_dict["orientation_loss"] = 0 + + if self.config.opacity_penalty: + loss_dict["opacity_loss"] = self.config.opacity_loss_mult * outputs["opacity_loss"].mean() + + if self.prompt != self.cur_prompt: + self.cur_prompt = self.prompt + self.text_embeddings.update_prompt( + base_prompt=self.cur_prompt, + top_prompt=self.cur_prompt + self.top_prompt, + side_prompt=self.cur_prompt + self.side_prompt, + back_prompt=self.cur_prompt + self.back_prompt, + front_prompt=self.cur_prompt + self.front_prompt, + ) + + text_embedding = self.text_embeddings.get_text_embedding( + vertical_angle=batch["vertical"], horizontal_angle=batch["central"] + ) + + train_output = ( + outputs["train_output"] + .view(1, int(outputs["train_output"].shape[0] ** 0.5), int(outputs["train_output"].shape[0] ** 0.5), 3) + .permute(0, 3, 1, 2) + ) + + sds_loss = self._diffusion_model.sds_loss( + text_embedding.to(self.diffusion_device), + train_output.to(self.diffusion_device), + guidance_scale=int(self.guidance_scale), + grad_scaler=self.grad_scaler, + ) + + loss_dict["sds_loss"] = sds_loss.to(self.device) + + if self.training: + loss_dict["distortion_loss"] = self.config.distortion_loss_mult * distortion_loss( + outputs["weights_list"], outputs["ray_samples_list"] + ) + loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss( + outputs["weights_list"], outputs["ray_samples_list"] + ) + return loss_dict + + def get_image_metrics_and_images( + self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor] + ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: + acc = colormaps.apply_colormap(outputs["accumulation"]) + depth = colormaps.apply_depth_colormap( + outputs["depth"], + accumulation=outputs["accumulation"], + ) + prop_depth_0 = colormaps.apply_depth_colormap( + outputs["prop_depth_0"], + accumulation=outputs["accumulation"], + ) + prop_depth_1 = colormaps.apply_depth_colormap( + outputs["prop_depth_1"], + accumulation=outputs["accumulation"], + ) + + metrics_dict = {} + images_dict = { + "img": outputs["rgb"], + "accumulation": acc, + "depth": depth, + "prop_depth_0": prop_depth_0, + "prop_depth_1": prop_depth_1, + "normals": outputs["normals"], + } + return metrics_dict, images_dict diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py index 1e1e0fab5..e9d30a9ab 100644 --- a/nerfstudio/pipelines/base_pipeline.py +++ b/nerfstudio/pipelines/base_pipeline.py @@ -40,8 +40,8 @@ from torch.cuda.amp.grad_scaler import GradScaler from nerfstudio.configs import base_config as cfg from nerfstudio.data.datamanagers.base_datamanager import ( DataManager, + DataManagerConfig, VanillaDataManager, - VanillaDataManagerConfig, ) from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes from nerfstudio.models.base_model import Model, ModelConfig @@ -205,7 +205,7 @@ class VanillaPipelineConfig(cfg.InstantiateConfig): _target: Type = field(default_factory=lambda: VanillaPipeline) """target class to instantiate""" - datamanager: VanillaDataManagerConfig = VanillaDataManagerConfig() + datamanager: DataManagerConfig = DataManagerConfig() """specifies the datamanager config""" model: ModelConfig = ModelConfig() """specifies the model config""" @@ -242,7 +242,7 @@ class VanillaPipeline(Pipeline): super().__init__() self.config = config self.test_mode = test_mode - self.datamanager: VanillaDataManager = config.datamanager.setup( + self.datamanager: DataManager = config.datamanager.setup( device=device, test_mode=test_mode, world_size=world_size, local_rank=local_rank ) self.datamanager.to(device) @@ -347,6 +347,7 @@ class VanillaPipeline(Pipeline): """ self.eval() metrics_dict_list = [] + assert isinstance(self.datamanager, VanillaDataManager) num_images = len(self.datamanager.fixed_indices_eval_dataloader) with Progress( TextColumn("[progress.description]{task.description}"), diff --git a/nerfstudio/scripts/train.py b/nerfstudio/scripts/train.py index cd5e73324..9ca18c71a 100644 --- a/nerfstudio/scripts/train.py +++ b/nerfstudio/scripts/train.py @@ -231,6 +231,10 @@ def main(config: TrainerConfig) -> None: CONSOLE.log("Using --data alias for --data.pipeline.datamanager.data") config.pipeline.datamanager.data = config.data + if config.prompt: + CONSOLE.log("Using --prompt alias for --data.pipeline.model.prompt") + config.pipeline.model.prompt = config.prompt + if config.load_config: CONSOLE.log(f"Loading pre-set config from: {config.load_config}") config = yaml.load(config.load_config.read_text(), Loader=yaml.Loader) diff --git a/nerfstudio/utils/eval_utils.py b/nerfstudio/utils/eval_utils.py index acd8bee18..13948678b 100644 --- a/nerfstudio/utils/eval_utils.py +++ b/nerfstudio/utils/eval_utils.py @@ -89,7 +89,6 @@ def eval_setup( assert isinstance(config, TrainerConfig) config.pipeline.datamanager._target = all_methods[config.method_name].pipeline.datamanager._target - if eval_num_rays_per_chunk: config.pipeline.model.eval_num_rays_per_chunk = eval_num_rays_per_chunk diff --git a/pyproject.toml b/pyproject.toml index 02972b0e9..b3452febe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,9 @@ dependencies = [ gen = [ "diffusers==0.16.1", "transformers==4.29.2", + "accelerate==0.19.0", + "bitsandbytes==0.39.0", + "sentencepiece==0.1.99", ] diff --git a/tests/pipelines/test_vanilla_pipeline.py b/tests/pipelines/test_vanilla_pipeline.py index 2218294b7..c27aa2069 100644 --- a/tests/pipelines/test_vanilla_pipeline.py +++ b/tests/pipelines/test_vanilla_pipeline.py @@ -11,10 +11,10 @@ from nerfstudio.data.datasets.base_dataset import DataparserOutputs, InputDatase from nerfstudio.pipelines.base_pipeline import ( Model, ModelConfig, - VanillaDataManagerConfig, VanillaPipeline, VanillaPipelineConfig, ) +from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManagerConfig class MockedDataManager: diff --git a/tests/test_train.py b/tests/test_train.py index a5749c022..d5d25c141 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -25,6 +25,7 @@ BLACKLIST = [ "phototourism", "depth-nerfacto", "neus", + "generfacto", "neus-facto", ]