diff --git a/docs/nerfology/methods/generfacto.md b/docs/nerfology/methods/generfacto.md
new file mode 100644
index 000000000..3323b9dfd
--- /dev/null
+++ b/docs/nerfology/methods/generfacto.md
@@ -0,0 +1,39 @@
+# Generfacto
+
+
Generate 3D models from text
+
+**Our model that combines generative 3D with our latest NeRF methods**
+
+## Installation
+
+First install nerfstudio dependencies. Then run:
+
+```bash
+pip install -e .[gen]
+```
+
+Two options for text to image diffusion models are provided: Stable Diffusion and DeepFloyd IF.
+We use Deepfloyd IF by default because it trains faster and produces better results. Using this model requires users to sign a license agreement for the model card of DeepFloyd IF, which can be found [here](https://huggingface.co/DeepFloyd/IF-I-XL-v1.0). Once the licensed is signed, log into Huggingface locally by running the following command:
+
+```bash
+huggingface-cli login
+```
+
+If you do not want to sign the license agreement, you can use the Stable Diffusion model (instructions below).
+
+## Running Generfacto
+
+Once installed, run:
+
+```bash
+ns-train generfacto --prompt "a high quality photo of a pineapple"
+```
+
+The first time you run this method, the diffusion model weights will be downloaded and cached
+from Hugging Face, which may take a couple minutes.
+
+Specify which diffusion model to use with the diffusion_model flag:
+
+```bash
+ns-train generfacto --pipeline.model.diffusion_model ["stablediffusion", "deepfloyd"]
+```
diff --git a/docs/nerfology/methods/index.md b/docs/nerfology/methods/index.md
index a11491c4d..8eac41800 100644
--- a/docs/nerfology/methods/index.md
+++ b/docs/nerfology/methods/index.md
@@ -36,6 +36,7 @@ The following methods are supported in nerfstudio:
NeRFPlayer
Tetra-NeRF
TensoRF
+ Generfacto
```
(own_method_docs)=
diff --git a/nerfstudio/cameras/camera_optimizers.py b/nerfstudio/cameras/camera_optimizers.py
index 899833f83..0ed2473f9 100644
--- a/nerfstudio/cameras/camera_optimizers.py
+++ b/nerfstudio/cameras/camera_optimizers.py
@@ -53,10 +53,10 @@ class CameraOptimizerConfig(InstantiateConfig):
orientation_noise_std: float = 0.0
"""Noise to add to initial orientations. Useful for debugging."""
- optimizer: OptimizerConfig = AdamOptimizerConfig(lr=6e-4, eps=1e-15)
+ optimizer: OptimizerConfig = field(default_factory=lambda: AdamOptimizerConfig(lr=6e-4, eps=1e-15))
"""ADAM parameters for camera optimization."""
- scheduler: SchedulerConfig = ExponentialDecaySchedulerConfig(max_steps=10000)
+ scheduler: SchedulerConfig = field(default_factory=lambda: ExponentialDecaySchedulerConfig(max_steps=10000))
"""Learning rate scheduler for camera optimizer.."""
param_group: tyro.conf.Suppress[str] = "camera_opt"
diff --git a/nerfstudio/configs/experiment_config.py b/nerfstudio/configs/experiment_config.py
index 2f57d50e4..c3a400dd1 100644
--- a/nerfstudio/configs/experiment_config.py
+++ b/nerfstudio/configs/experiment_config.py
@@ -72,6 +72,8 @@ class ExperimentConfig(InstantiateConfig):
"""Which visualizer to use."""
data: Optional[Path] = None
"""Alias for --pipeline.datamanager.data"""
+ prompt: Optional[str] = None
+ """Alias for --pipeline.model.prompt"""
relative_model_dir: Path = Path("nerfstudio_models/")
"""Relative path to save all checkpoints."""
diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py
index e0a270ba9..b8ec7fede 100644
--- a/nerfstudio/configs/method_configs.py
+++ b/nerfstudio/configs/method_configs.py
@@ -26,7 +26,10 @@ import tyro
from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig
from nerfstudio.configs.base_config import ViewerConfig
from nerfstudio.configs.external_methods import get_external_methods
+
+from nerfstudio.data.datamanagers.random_cameras_datamanager import RandomCamerasDataManagerConfig
from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager, VanillaDataManagerConfig
+
from nerfstudio.data.dataparsers.blender_dataparser import BlenderDataParserConfig
from nerfstudio.data.dataparsers.dnerf_dataparser import DNeRFDataParserConfig
from nerfstudio.data.dataparsers.instant_ngp_dataparser import InstantNGPDataParserConfig
@@ -47,6 +50,7 @@ from nerfstudio.engine.trainer import TrainerConfig
from nerfstudio.field_components.temporal_distortions import TemporalDistortionKind
from nerfstudio.fields.sdf_field import SDFFieldConfig
from nerfstudio.models.depth_nerfacto import DepthNerfactoModelConfig
+from nerfstudio.models.generfacto import GenerfactoModelConfig
from nerfstudio.models.instant_ngp import InstantNGPModelConfig
from nerfstudio.models.mipnerf import MipNerfModel
from nerfstudio.models.nerfacto import NerfactoModelConfig
@@ -72,6 +76,7 @@ descriptions = {
"tensorf": "tensorf",
"dnerf": "Dynamic-NeRF model. (slow)",
"phototourism": "Uses the Phototourism data.",
+ "generfacto": "Generative Text to NeRF model",
"neus": "Implementation of NeuS. (slow)",
"neus-facto": "Implementation of NeuS-Facto. (slow)",
}
@@ -483,6 +488,51 @@ method_configs["phototourism"] = TrainerConfig(
vis="viewer",
)
+method_configs["generfacto"] = TrainerConfig(
+ method_name="generfacto",
+ experiment_name="",
+ steps_per_eval_batch=50,
+ steps_per_eval_image=50,
+ steps_per_save=200,
+ max_num_iterations=30000,
+ mixed_precision=True,
+ pipeline=VanillaPipelineConfig(
+ datamanager=RandomCamerasDataManagerConfig(
+ horizontal_rotation_warmup=3000,
+ ),
+ model=GenerfactoModelConfig(
+ eval_num_rays_per_chunk=1 << 15,
+ distortion_loss_mult=1.0,
+ interlevel_loss_mult=100.0,
+ max_res=256,
+ sphere_collider=True,
+ initialize_density=True,
+ taper_range=(0, 2000),
+ random_background=True,
+ proposal_warmup=2000,
+ proposal_update_every=0,
+ proposal_weights_anneal_max_num_iters=2000,
+ start_lambertian_training=500,
+ start_normals_training=2000,
+ opacity_loss_mult=0.001,
+ positional_prompting="discrete",
+ guidance_scale=25,
+ ),
+ ),
+ optimizers={
+ "proposal_networks": {
+ "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15),
+ "scheduler": None,
+ },
+ "fields": {
+ "optimizer": AdamOptimizerConfig(lr=5e-4, eps=1e-15),
+ "scheduler": None,
+ },
+ },
+ viewer=ViewerConfig(),
+ vis="viewer",
+)
+
method_configs["neus"] = TrainerConfig(
method_name="neus",
steps_per_eval_image=500,
diff --git a/nerfstudio/data/datamanagers/random_cameras_datamanager.py b/nerfstudio/data/datamanagers/random_cameras_datamanager.py
new file mode 100644
index 000000000..4269b428a
--- /dev/null
+++ b/nerfstudio/data/datamanagers/random_cameras_datamanager.py
@@ -0,0 +1,315 @@
+# Copyright 2022 The Nerfstudio Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Data manager without input images, only random camera poses
+"""
+
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass, field
+from typing import Dict, List, Tuple, Type, Union
+
+import torch
+from rich.progress import Console
+from torch.nn import Parameter
+from torch import Tensor
+from typing_extensions import Literal
+
+from nerfstudio.cameras.cameras import Cameras
+from nerfstudio.cameras.rays import RayBundle
+from nerfstudio.configs.config_utils import to_immutable_dict
+from nerfstudio.data.datamanagers.base_datamanager import DataManager, DataManagerConfig
+from nerfstudio.data.datasets.base_dataset import InputDataset
+from nerfstudio.data.scene_box import SceneBox
+from nerfstudio.data.utils.dataloaders import RandIndicesEvalDataloader
+
+CONSOLE = Console(width=120)
+
+
+class TrivialDataset(InputDataset):
+ """A trivial dataset with blank images for the viewer"""
+
+ # pylint: disable=super-init-not-called
+ def __init__(self, cameras: Cameras):
+ self.size = cameras.size
+ self.cameras = cameras
+ self.alpha_color = None
+ self.scene_box = SceneBox(torch.Tensor([[-1, -1, -1], [1, 1, 1]]))
+ self.mask_filenames = None
+ self.metadata = to_immutable_dict({})
+
+ def __len__(self):
+ return self.size
+
+ def __getitem__(self, index: int) -> Dict:
+ return {
+ "image": torch.cat([torch.ones(128, 256, 3), torch.zeros(128, 256, 3)], dim=0),
+ "image_idx": index,
+ }
+
+
+def random_train_pose(
+ size: int,
+ resolution: int,
+ device: Union[torch.device, str],
+ radius_mean: float = 1.0,
+ radius_std: float = 0.1,
+ central_rotation_range: Tuple[float, float] = (0, 360),
+ vertical_rotation_range: Tuple[float, float] = (-90, 0),
+ focal_range: Tuple[float, float] = (0.75, 1.35),
+ jitter_std: float = 0.01,
+ center: Tuple[float, float, float] = (0, 0, 0),
+) -> Tuple[Cameras, Tensor, Tensor]:
+ """generate random poses from an orbit camera
+ Args:
+ size: batch size of generated poses.
+ device: where to allocate the output.
+ radius_mean: mean radius of the orbit camera.
+ radius_std: standard deviation of the radius of the orbit camera.
+ central_rotation_range: amount that we rotate around the center of the object
+ vertical_rotation_range: amount that we allow the cameras to pan up and down from horizontal
+ focal_range: focal length range
+ jitter_std: standard deviation of the jitter added to the camera position
+ center: center of the object
+ Return:
+ poses: [size, 4, 4]
+ """
+
+ vertical_rotation_range = (
+ vertical_rotation_range[0] + 90,
+ vertical_rotation_range[1] + 90,
+ )
+ # This is the uniform sample on the part of the sphere we care about where 0 = 0 degrees and 1 = 360 degrees
+ sampled_uniform = (
+ torch.rand(size) * (vertical_rotation_range[1] - vertical_rotation_range[0]) + vertical_rotation_range[0]
+ ) / 180
+ vertical_rotation = torch.arccos(1 - 2 * sampled_uniform)
+ central_rotation = torch.deg2rad(
+ torch.rand(size) * (central_rotation_range[1] - central_rotation_range[0]) + central_rotation_range[0]
+ )
+
+ c_cos = torch.cos(central_rotation)
+ c_sin = torch.sin(central_rotation)
+ v_cos = torch.cos(vertical_rotation)
+ v_sin = torch.sin(vertical_rotation)
+ zeros = torch.zeros_like(central_rotation)
+ ones = torch.ones_like(central_rotation)
+
+ rot_z = torch.stack(
+ [
+ torch.stack([c_cos, -c_sin, zeros], dim=-1),
+ torch.stack([c_sin, c_cos, zeros], dim=-1),
+ torch.stack([zeros, zeros, ones], dim=-1),
+ ],
+ dim=-2,
+ )
+
+ rot_x = torch.stack(
+ [
+ torch.stack([ones, zeros, zeros], dim=-1),
+ torch.stack([zeros, v_cos, -v_sin], dim=-1),
+ torch.stack([zeros, v_sin, v_cos], dim=-1),
+ ],
+ dim=-2,
+ )
+
+ # Default directions are facing in the -z direction, so origins should face opposite way
+ origins = torch.stack([torch.tensor([0, 0, 1])] * size, dim=0)
+ origins = (origins * radius_mean) + (origins * (torch.randn((origins.shape)) * radius_std))
+ R = torch.bmm(rot_z, rot_x) # Want to have Rx @ Ry @ origin
+ t = (
+ torch.bmm(R, origins.unsqueeze(-1))
+ + torch.randn((size, 3, 1)) * jitter_std
+ + torch.tensor(center)[None, :, None]
+ )
+ camera_to_worlds = torch.cat([R, t], dim=-1)
+
+ focals = torch.rand(size) * (focal_range[1] - focal_range[0]) + focal_range[0]
+
+ cameras = Cameras(
+ camera_to_worlds=camera_to_worlds,
+ fx=focals * resolution,
+ fy=focals * resolution,
+ cx=resolution / 2,
+ cy=resolution / 2,
+ ).to(device)
+
+ return cameras, torch.rad2deg(vertical_rotation), torch.rad2deg(central_rotation)
+
+
+@dataclass
+class RandomCamerasDataManagerConfig(DataManagerConfig):
+ """Configuration for data manager that does not load from a dataset. Instead, it generates random poses."""
+
+ _target: Type = field(default_factory=lambda: RandomCamerasDataManager)
+ train_resolution: int = 64
+ """Training resolution"""
+ eval_resolution: int = 64
+ """Evaluation resolution"""
+ num_eval_angles: int = 256
+ """Number of evaluation angles"""
+ train_images_per_batch: int = 1
+ """Number of images per batch for training"""
+ eval_images_per_batch: int = 1
+ """Number of images per batch for evaluation"""
+ radius_mean: float = 2.5
+ """Mean radius of camera orbit"""
+ radius_std: float = 0.1
+ """Std of radius of camera orbit"""
+ focal_range: Tuple[float, float] = (0.7, 1.35)
+ """Range of focal length"""
+ vertical_rotation_range: Tuple[float, float] = (-90, 0)
+ """Range of vertical rotation"""
+ jitter_std: float = 0.05
+ """Std of camera direction jitter, so we don't just point the cameras towards the center every time"""
+ center: Tuple[float, float, float] = (0, 0, 0)
+ """Center coordinate of the camera sphere"""
+ horizontal_rotation_warmup: int = 0
+ """How many steps until the full horizontal rotation range is used"""
+
+
+class RandomCamerasDataManager(DataManager): # pylint: disable=abstract-method
+ """Basic stored data manager implementation.
+
+ This is pretty much a port over from our old dataloading utilities, and is a little jank
+ under the hood. We may clean this up a little bit under the hood with more standard dataloading
+ components that can be strung together, but it can be just used as a black box for now since
+ only the constructor is likely to change in the future, or maybe passing in step number to the
+ next_train and next_eval functions.
+
+ Args:
+ config: the DataManagerConfig used to instantiate class
+ """
+
+ config: RandomCamerasDataManagerConfig
+
+ # pylint: disable=super-init-not-called
+ def __init__(
+ self,
+ config: RandomCamerasDataManagerConfig,
+ device: Union[torch.device, str] = "cpu",
+ test_mode: Literal["test", "val", "inference"] = "val",
+ world_size: int = 1,
+ local_rank: int = 0,
+ **kwargs, # pylint: disable=unused-argument
+ ):
+ self.config = config
+ self.device = device
+ self.world_size = world_size
+ self.local_rank = local_rank
+ self.sampler = None
+ self.test_mode = test_mode
+ self.test_split = "test" if test_mode in ["test", "inference"] else "val"
+
+ if self.config.data is not None:
+ CONSOLE.print("[red] --data should not be used with the RandomCamerasDataManager[/red]")
+ sys.exit(1)
+
+ cameras, _, _ = random_train_pose(
+ self.config.num_eval_angles,
+ self.config.eval_resolution,
+ device=self.device,
+ radius_mean=self.config.radius_mean,
+ radius_std=self.config.radius_std,
+ focal_range=self.config.focal_range,
+ central_rotation_range=(-180, 180),
+ vertical_rotation_range=self.config.vertical_rotation_range,
+ jitter_std=self.config.jitter_std,
+ center=self.config.center,
+ )
+
+ self.train_dataset = TrivialDataset(cameras)
+ self.eval_dataset = TrivialDataset(cameras)
+
+ self.eval_dataloader = RandIndicesEvalDataloader(
+ input_dataset=self.eval_dataset,
+ device=self.device,
+ num_workers=self.world_size * 4,
+ )
+
+ # pylint: disable=non-parent-init-called
+ DataManager.__init__(self)
+
+ def next_train(self, step: int) -> Tuple[RayBundle, Dict]:
+ """Returns the next batch of data from the train dataloader."""
+
+ self.train_count += 1
+ horizontal_range = min((step / max(1, self.config.horizontal_rotation_warmup)), 1) * 180
+
+ cameras, vertical_rotation, central_rotation = random_train_pose(
+ self.config.train_images_per_batch,
+ self.config.train_resolution,
+ device=self.device,
+ radius_mean=self.config.radius_mean,
+ radius_std=self.config.radius_std,
+ focal_range=self.config.focal_range,
+ vertical_rotation_range=self.config.vertical_rotation_range,
+ jitter_std=self.config.jitter_std,
+ center=self.config.center,
+ central_rotation_range=(-horizontal_range, horizontal_range),
+ )
+ ray_bundle = cameras.generate_rays(torch.tensor(list(range(self.config.train_images_per_batch)))).flatten()
+
+ return ray_bundle, {
+ "vertical": vertical_rotation,
+ "central": central_rotation,
+ "initialization": True,
+ }
+
+ def next_eval(self, step: int) -> Tuple[RayBundle, Dict]:
+ """Returns the next batch of data from the eval dataloader."""
+ self.eval_count += 1
+
+ cameras, vertical_rotation, central_rotation = random_train_pose(
+ self.config.eval_images_per_batch,
+ self.config.eval_resolution,
+ device=self.device,
+ radius_mean=self.config.radius_mean,
+ radius_std=self.config.radius_std,
+ focal_range=self.config.focal_range,
+ vertical_rotation_range=self.config.vertical_rotation_range,
+ jitter_std=self.config.jitter_std,
+ center=self.config.center,
+ )
+ ray_bundle = cameras.generate_rays(
+ torch.tensor([[i] for i in range(self.config.train_images_per_batch)])
+ ).flatten()
+
+ return ray_bundle, {"vertical": vertical_rotation, "central": central_rotation}
+
+ def next_eval_image(self, step: int) -> Tuple[int, RayBundle, Dict]:
+ for camera_ray_bundle, batch in self.eval_dataloader:
+ assert camera_ray_bundle.camera_indices is not None
+ image_idx = int(camera_ray_bundle.camera_indices[0, 0, 0])
+ return image_idx, camera_ray_bundle, batch
+ raise ValueError("No more eval images")
+
+ def get_train_rays_per_batch(self) -> int:
+ return self.config.train_resolution**2
+
+ def get_eval_rays_per_batch(self) -> int:
+ return self.config.eval_resolution**2
+
+ def get_param_groups(
+ self,
+ ) -> Dict[str, List[Parameter]]: # pylint: disable=no-self-use
+ """Get the param groups for the data manager.
+ Returns:
+ A list of dictionaries containing the data manager's param groups.
+ """
+ param_groups = {}
+
+ return param_groups
diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py
index afc0d94f3..4afb3a69e 100644
--- a/nerfstudio/engine/trainer.py
+++ b/nerfstudio/engine/trainer.py
@@ -34,6 +34,7 @@ from torch.cuda.amp.grad_scaler import GradScaler
from nerfstudio.configs.experiment_config import ExperimentConfig
from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes, TrainingCallbackLocation
+from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager
from nerfstudio.engine.optimizers import Optimizers
from nerfstudio.pipelines.base_pipeline import VanillaPipeline
from nerfstudio.utils import profiler, writer
@@ -224,9 +225,11 @@ class Trainer:
"""Train the model."""
assert self.pipeline.datamanager.train_dataset is not None, "Missing DatsetInputs"
- self.pipeline.datamanager.train_dataparser_outputs.save_dataparser_transform(
- self.base_dir / "dataparser_transforms.json"
- )
+ # don't want to call save_dataparser_transform if pipeline's datamanager does not have a dataparser
+ if isinstance(self.pipeline.datamanager, VanillaDataManager):
+ self.pipeline.datamanager.train_dataparser_outputs.save_dataparser_transform(
+ self.base_dir / "dataparser_transforms.json"
+ )
self._init_viewer_state()
with TimeWriter(writer, EventName.TOTAL_TRAIN_TIME):
diff --git a/nerfstudio/field_components/field_heads.py b/nerfstudio/field_components/field_heads.py
index ed199b4b9..44df9f602 100644
--- a/nerfstudio/field_components/field_heads.py
+++ b/nerfstudio/field_components/field_heads.py
@@ -34,6 +34,7 @@ class FieldHeadNames(Enum):
NORMALS = "normals"
PRED_NORMALS = "pred_normals"
UNCERTAINTY = "uncertainty"
+ BACKGROUND_RGB = "background_rgb"
TRANSIENT_RGB = "transient_rgb"
TRANSIENT_DENSITY = "transient_density"
SEMANTICS = "semantics"
diff --git a/nerfstudio/fields/generfacto_field.py b/nerfstudio/fields/generfacto_field.py
new file mode 100644
index 000000000..d72f5c263
--- /dev/null
+++ b/nerfstudio/fields/generfacto_field.py
@@ -0,0 +1,169 @@
+# Copyright 2022 The Nerfstudio Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Field for Generfacto model
+"""
+
+
+from typing import Optional, Tuple, Dict, Literal
+
+import numpy as np
+import torch
+from torch.nn.parameter import Parameter
+from jaxtyping import Float
+from torch import Tensor, nn
+
+from nerfstudio.field_components.mlp import MLP
+from nerfstudio.cameras.rays import RayBundle, RaySamples
+from nerfstudio.data.scene_box import SceneBox
+from nerfstudio.field_components.activations import trunc_exp
+from nerfstudio.field_components.field_heads import FieldHeadNames
+from nerfstudio.field_components.encodings import HashEncoding, SHEncoding
+from nerfstudio.fields.base_field import Field
+
+
+def get_normalized_directions(directions: Float[Tensor, "bs 3"]):
+ """SH encoding must be in the range [0, 1]
+
+ Args:
+ directions: batch of directions
+ """
+ return (directions + 1.0) / 2.0
+
+
+class GenerfactoField(Field):
+ """Generfacto Field that uses TCNN
+
+ Args:
+ aabb: parameters of scene aabb bounds
+ num_layers: number of hidden layers
+ hidden_dim: dimension of hidden layers
+ geo_feat_dim: output geo feat dimensions
+ num_levels: number of levels of the hashmap for the base mlp
+ max_res: maximum resolution of the hashmap for the base mlp
+ log2_hashmap_size: size of the hashmap for the base mlp
+ num_layers_color: number of hidden layers for color network
+ hidden_dim_color: dimension of hidden layers for color network
+ """
+
+ def __init__(
+ self,
+ aabb: Tensor,
+ num_layers: int = 2,
+ hidden_dim: int = 64,
+ geo_feat_dim: int = 15,
+ num_levels: int = 16,
+ max_res: int = 256,
+ log2_hashmap_size: int = 19,
+ num_layers_color: int = 3,
+ hidden_dim_color: int = 64,
+ implementation: Literal["tcnn", "torch"] = "tcnn",
+ ) -> None:
+ super().__init__()
+
+ self.aabb = Parameter(aabb, requires_grad=False)
+ self.geo_feat_dim = geo_feat_dim
+
+ base_res = 16
+ features_per_level = 2
+ np.exp((np.log(max_res) - np.log(base_res)) / (num_levels - 1))
+
+ self.direction_encoding = SHEncoding(
+ levels=4,
+ implementation=implementation,
+ )
+
+ encoder = HashEncoding(
+ num_levels=num_levels,
+ min_res=base_res,
+ max_res=max_res,
+ log2_hashmap_size=log2_hashmap_size,
+ features_per_level=features_per_level,
+ implementation=implementation,
+ )
+ network = MLP(
+ in_dim=encoder.get_out_dim(),
+ num_layers=num_layers,
+ layer_width=hidden_dim,
+ out_dim=1 + self.geo_feat_dim,
+ activation=nn.ReLU(),
+ out_activation=None,
+ implementation=implementation,
+ )
+ self.mlp_base = torch.nn.Sequential(encoder, network)
+
+ self.mlp_background_color = MLP(
+ in_dim=self.direction_encoding.get_out_dim(),
+ num_layers=2,
+ layer_width=32,
+ out_dim=3,
+ activation=nn.ReLU(),
+ out_activation=nn.Sigmoid(),
+ implementation=implementation,
+ )
+
+ self.mlp_head = MLP(
+ in_dim=self.geo_feat_dim,
+ num_layers=num_layers_color,
+ layer_width=hidden_dim_color,
+ out_dim=3,
+ activation=nn.ReLU(),
+ out_activation=nn.Sigmoid(),
+ implementation=implementation,
+ )
+
+ def get_density(self, ray_samples: RaySamples) -> Tuple[Tensor, Tensor]:
+ """Computes and returns the densities."""
+ positions = SceneBox.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb)
+ self._sample_locations = positions
+ if not self._sample_locations.requires_grad:
+ self._sample_locations.requires_grad = True
+ positions_flat = positions.view(-1, 3)
+ h = self.mlp_base(positions_flat).view(*ray_samples.frustums.shape, -1)
+ density_before_activation, base_mlp_out = torch.split(h, [1, self.geo_feat_dim], dim=-1)
+ self._density_before_activation = density_before_activation
+
+ # Rectifying the density with an exponential is much more stable than a ReLU or
+ # softplus, because it enables high post-activation (float32) density outputs
+ # from smaller internal (float16) parameters.
+ density = trunc_exp(density_before_activation.to(positions))
+ return density, base_mlp_out
+
+ def get_background_rgb(self, ray_bundle: RayBundle) -> Tensor:
+ """Predicts background colors at infinity."""
+ directions = get_normalized_directions(ray_bundle.directions)
+
+ outputs_shape = ray_bundle.directions.shape[:-1]
+ directions_flat = self.direction_encoding(directions.view(-1, 3))
+ background_rgb = self.mlp_background_color(directions_flat).view(*outputs_shape, -1).to(directions)
+
+ return background_rgb
+
+ def get_outputs(
+ self, ray_samples: RaySamples, density_embedding: Optional[Tensor] = None
+ ) -> Dict[FieldHeadNames, Tensor]:
+ assert density_embedding is not None
+ outputs = {}
+
+ directions = get_normalized_directions(ray_samples.frustums.directions)
+
+ outputs_shape = ray_samples.frustums.directions.shape[:-1]
+
+ h = density_embedding.view(-1, self.geo_feat_dim)
+
+ rgb = self.mlp_head(h).view(*outputs_shape, -1).to(directions)
+ outputs.update({FieldHeadNames.RGB: rgb})
+
+ return outputs
diff --git a/nerfstudio/generative/deepfloyd.py b/nerfstudio/generative/deepfloyd.py
index 982be0879..72c6cc373 100644
--- a/nerfstudio/generative/deepfloyd.py
+++ b/nerfstudio/generative/deepfloyd.py
@@ -14,21 +14,30 @@
import gc
from pathlib import Path
-from typing import List, Optional, Union, cast
+from typing import List, Optional, Union
import torch
import torch.nn.functional as F
import tyro
-from diffusers import IFPipeline
-from diffusers.pipelines.deepfloyd_if import IFPipelineOutput
+
from jaxtyping import Float
from PIL import Image
-from torch import FloatTensor, Generator, Tensor, nn
+from torch import Generator, Tensor, nn
from torch.cuda.amp.grad_scaler import GradScaler
-from transformers import T5EncoderModel
-from nerfstudio.generative.utils import _SDSGradient
+from nerfstudio.generative.utils import CatchMissingPackages
+
+try:
+ from diffusers import IFPipeline as IFOrig
+ from diffusers.pipelines.deepfloyd_if import IFPipelineOutput as IFOutputOrig
+
+ from diffusers import IFPipeline, DiffusionPipeline
+ from diffusers.pipelines.deepfloyd_if import IFPipelineOutput
+ from transformers import T5EncoderModel
+
+except ImportError:
+ IFPipeline = IFPipelineOutput = T5EncoderModel = CatchMissingPackages()
IMG_DIM = 64
@@ -61,7 +70,7 @@ class DeepFloyd(nn.Module):
variant="fp16",
torch_dtype=torch.float16,
)
- assert isinstance(self.pipe, IFPipeline)
+ assert isinstance(self.pipe, DiffusionPipeline)
self.pipe = self.pipe.to(self.device)
self.pipe.enable_attention_slicing(1)
@@ -96,7 +105,7 @@ class DeepFloyd(nn.Module):
variant="fp16",
torch_dtype=torch.float16,
)
- assert isinstance(self.pipe, IFPipeline)
+ assert isinstance(self.pipe, DiffusionPipeline)
self.pipe = self.pipe.to(self.device)
self.pipe.enable_attention_slicing(1)
@@ -122,12 +131,12 @@ class DeepFloyd(nn.Module):
prompt = [prompt] if isinstance(prompt, str) else prompt
negative_prompt = [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
- assert isinstance(self.pipe, IFPipeline)
+ assert isinstance(self.pipe, DiffusionPipeline)
with torch.no_grad():
prompt_embeds, negative_embeds = self.pipe.encode_prompt(prompt, negative_prompt=negative_prompt)
- assert isinstance(negative_embeds, FloatTensor)
- assert isinstance(prompt_embeds, FloatTensor)
+ assert isinstance(negative_embeds, Tensor)
+ assert isinstance(prompt_embeds, Tensor)
return torch.cat([negative_embeds, prompt_embeds])
def sds_loss(
@@ -146,40 +155,37 @@ class DeepFloyd(nn.Module):
Returns:
The loss
"""
- with torch.autocast(device_type="cuda", enabled=False):
- image = F.interpolate(image.half(), (IMG_DIM, IMG_DIM), mode="bilinear", align_corners=False)
- t = torch.randint(self.min_step, self.max_step + 1, [1], dtype=torch.long, device=self.device)
+ image = F.interpolate(image.half(), (IMG_DIM, IMG_DIM), mode="bilinear", align_corners=False)
+ t = torch.randint(self.min_step, self.max_step + 1, [1], dtype=torch.long, device=self.device)
- # predict the noise residual with unet, NO grad!
- with torch.no_grad():
- # add noise
- noise = torch.randn_like(image)
- image_noisy = self.scheduler.add_noise(image, noise, t) # type: ignore
- # pred noise
- image_model_input = torch.cat((image_noisy,) * 2)
- noise_pred = self.unet(image_model_input, t, encoder_hidden_states=text_embeddings).sample
+ # predict the noise residual with unet, NO grad!
+ with torch.no_grad():
+ # add noise
+ noise = torch.randn_like(image)
+ image_noisy = self.scheduler.add_noise(image, noise, t) # type: ignore
+ # pred noise
+ image_model_input = torch.cat((image_noisy,) * 2)
+ noise_pred = self.unet(image_model_input, t, encoder_hidden_states=text_embeddings).sample
- # perform guidance
- noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
- noise_pred_text, predicted_variance = noise_pred_text.split(3, dim=1)
- noise_pred_uncond, _ = noise_pred_uncond.split(3, dim=1)
+ # perform guidance
+ noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
+ noise_pred_text, predicted_variance = noise_pred_text.split(3, dim=1)
+ noise_pred_uncond, _ = noise_pred_uncond.split(3, dim=1)
- noise_pred = noise_pred_text + guidance_scale * (noise_pred_text - noise_pred_uncond)
+ noise_pred = noise_pred_text + guidance_scale * (noise_pred_text - noise_pred_uncond)
- # w(t), sigma_t^2
- w = 1 - self.alphas[t]
+ # w(t), sigma_t^2
+ w = 1 - self.alphas[t]
- grad = w * (noise_pred - noise)
- grad = torch.nan_to_num(grad)
+ grad = w * (noise_pred - noise)
+ grad = torch.nan_to_num(grad)
- if grad_scaler is not None:
- loss = cast(Tensor, _SDSGradient.apply(grad_scaler.scale(image), grad))
- else:
- loss = cast(Tensor, _SDSGradient.apply(image, grad))
+ target = (image - grad).detach()
+ loss = 0.5 * F.mse_loss(image, target, reduction="sum") / image.shape[0]
return loss
- def prompt_to_img(
+ def prompt_to_image(
self,
prompts: Union[str, List[str]],
negative_prompts: Union[str, List[str]] = "",
@@ -201,16 +207,17 @@ class DeepFloyd(nn.Module):
prompts = [prompts] if isinstance(prompts, str) else prompts
negative_prompts = [negative_prompts] if isinstance(negative_prompts, str) else negative_prompts
-
- assert isinstance(self.pipe, IFPipeline)
+ assert isinstance(self.pipe, DiffusionPipeline)
prompt_embeds, negative_embeds = self.pipe.encode_prompt(prompts, negative_prompt=negative_prompts)
+
+ assert isinstance(self.pipe, IFOrig)
model_output = self.pipe(
prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_embeds, generator=generator
)
- assert isinstance(model_output, IFPipelineOutput)
- image = model_output.images[0]
+ assert isinstance(model_output, IFOutputOrig)
+ output_image = model_output.images[0]
- return image
+ return output_image
def generate_image(
@@ -228,7 +235,7 @@ def generate_image(
cuda_device = torch.device("cuda")
with torch.no_grad():
df = DeepFloyd(cuda_device)
- img = df.prompt_to_img(prompt, negative, generator, steps)
+ img = df.prompt_to_image(prompt, negative, generator, steps)
img.save(save_path)
diff --git a/nerfstudio/generative/positional_text_embeddings.py b/nerfstudio/generative/positional_text_embeddings.py
new file mode 100644
index 000000000..4ba62b024
--- /dev/null
+++ b/nerfstudio/generative/positional_text_embeddings.py
@@ -0,0 +1,120 @@
+# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utility helper functions for generative 3D models"""
+
+import torch
+from torch import Tensor
+from jaxtyping import Float
+from typing_extensions import Literal
+from typing import Union
+
+from nerfstudio.generative.stable_diffusion import StableDiffusion
+from nerfstudio.generative.deepfloyd import DeepFloyd
+
+
+class PositionalTextEmbeddings:
+ """Postional Prompts. Currently the following location based text embeddings are supported:
+ - discrete: Choose the embedding closest to the camera position
+ - interpolated: Interpolate between the embeddings based on the camera position
+ - off: Don't modify the text embedding based on the camera position
+
+ Args:
+ base_prompt: Prompt for base view
+ top_prompt: Prompt for top view
+ side_prompt: Prompt for side view
+ back_prompt: Prompt for back view
+ front_prompt: Prompt for front view
+ diffusion_model: Instance of StableDiffusion
+ positional_prompting: how to incorporate position into prompt.
+ """
+
+ def __init__(
+ self,
+ base_prompt: str,
+ top_prompt: str,
+ side_prompt: str,
+ back_prompt: str,
+ front_prompt: str,
+ diffusion_model: Union[StableDiffusion, DeepFloyd],
+ positional_prompting: Literal["discrete", "interpolated", "off"] = "discrete",
+ ):
+ self.positional_prompting = positional_prompting
+ self.diffusion_device = diffusion_model.device
+ self.diffusion = diffusion_model
+ self.update_prompt(base_prompt, top_prompt, side_prompt, back_prompt, front_prompt)
+
+ def update_prompt(self, base_prompt: str, top_prompt: str, side_prompt: str, back_prompt: str, front_prompt: str):
+ """Update the text embeddings based on the new prompts.
+
+ Args:
+ base_prompt: Prompt for base view
+ top_prompt: Prompt for top view
+ side_prompt: Prompt for side view
+ back_prompt: Prompt for back view
+ front_prompt: Prompt for front view
+ """
+ self.base_embed = self.diffusion.get_text_embeds(base_prompt, "")
+ self.top_embed = self.diffusion.get_text_embeds(top_prompt, "")
+ self.side_embed = self.diffusion.get_text_embeds(side_prompt, "")
+ self.back_embed = self.diffusion.get_text_embeds(back_prompt, "")
+ self.front_embed = self.diffusion.get_text_embeds(front_prompt, "")
+
+ if isinstance(self.diffusion, DeepFloyd):
+ self.diffusion.delete_text_encoder()
+
+ def get_text_embedding(
+ self, vertical_angle: Float[Tensor, "1"], horizontal_angle: Float[Tensor, "1"]
+ ) -> Float[Tensor, "2 max_length embed_dim"]:
+ """Get text embedding based on the position of the camera relative to the scene.
+ This trick is used in Dreamfusion (https://dreamfusion3d.github.io/).
+
+ Args:
+ vertical_angle: vertical angle of the camera
+ horizonal_angle: horizonal angle of the camera
+ """
+ # set horizontal_angle between 0, 360
+ horizontal_angle = torch.fmod(horizontal_angle, 360)
+ horizontal_angle = torch.where(horizontal_angle < 0, horizontal_angle + 360, horizontal_angle)
+
+ if self.positional_prompting == "discrete":
+ if vertical_angle < 40:
+ text_embedding = self.top_embed
+ elif 315 < horizontal_angle or horizontal_angle <= 45:
+ text_embedding = self.front_embed
+ elif 45 < horizontal_angle <= 135:
+ text_embedding = self.side_embed
+ elif 135 < horizontal_angle <= 225:
+ text_embedding = self.back_embed
+ else: # horizontal_angle > 225 and horizontal_angle <= 315:
+ text_embedding = self.side_embed
+ elif self.positional_prompting == "interpolated":
+ horiz = horizontal_angle.detach().numpy()[0]
+ vert = max(vertical_angle.detach().numpy()[0], 0)
+
+ if 0 < horizontal_angle <= 90:
+ text_embedding = (horiz) * self.side_embed + (90 - horiz) * self.front_embed
+ elif 90 < horizontal_angle <= 180:
+ text_embedding = (horiz - 90) * self.back_embed + (180 - horiz) * self.side_embed
+ elif 180 < horizontal_angle <= 270:
+ text_embedding = (horiz - 180) * self.side_embed + (270 - horiz) * self.back_embed
+ else: # 270 < horizontal_angle <= 360:
+ text_embedding = (horiz - 270) * self.front_embed + (360 - horiz) * self.side_embed
+
+ text_embedding = text_embedding / 90.0
+ text_embedding = (vert * text_embedding + (90 - vert) * self.top_embed) / 90.0
+ else:
+ text_embedding = self.base_embed
+
+ return text_embedding
diff --git a/nerfstudio/generative/stable_diffusion.py b/nerfstudio/generative/stable_diffusion.py
index 6202a9046..7e7f3dac4 100644
--- a/nerfstudio/generative/stable_diffusion.py
+++ b/nerfstudio/generative/stable_diffusion.py
@@ -16,10 +16,9 @@
# Modified from https://github.com/ashawkey/stable-dreamfusion/blob/main/nerf/sd.py
-import sys
-from dataclasses import dataclass
from pathlib import Path
-from typing import List, Optional, Union, cast
+from typing import List, Optional, Union
+from nerfstudio.utils.rich_utils import CONSOLE
import mediapy
import numpy as np
@@ -30,22 +29,16 @@ from jaxtyping import Float
from torch import Tensor, nn
from torch.cuda.amp.grad_scaler import GradScaler
-from nerfstudio.utils.rich_utils import CONSOLE
-
-from nerfstudio.generative.utils import _SDSGradient
+from nerfstudio.generative.utils import CatchMissingPackages
try:
- from diffusers import PNDMScheduler, StableDiffusionPipeline
- from transformers import logging
+ from diffusers import PNDMScheduler, StableDiffusionPipeline, DiffusionPipeline
except ImportError:
- CONSOLE.print("[bold red]Missing Stable Diffusion packages.")
- CONSOLE.print(r"Install using [yellow]pip install nerfstudio\[gen][/yellow]")
- CONSOLE.print(r"or [yellow]pip install -e .\[gen][/yellow] if installing from source.")
- sys.exit(1)
+ PNDMScheduler = StableDiffusionPipeline = CatchMissingPackages()
+
-logging.set_verbosity_error()
IMG_DIM = 512
CONST_SCALE = 0.18215
SD_IDENTIFIERS = {
@@ -55,13 +48,6 @@ SD_IDENTIFIERS = {
}
-@dataclass
-class UNet2DConditionOutput:
- """Class to hold traced model"""
-
- sample: torch.FloatTensor
-
-
class StableDiffusion(nn.Module):
"""Stable Diffusion implementation
Args:
@@ -88,7 +74,8 @@ class StableDiffusion(nn.Module):
sd_id = SD_IDENTIFIERS[version]
pipe = StableDiffusionPipeline.from_pretrained(sd_id, torch_dtype=torch.float16)
- assert isinstance(pipe, StableDiffusionPipeline)
+
+ assert isinstance(pipe, DiffusionPipeline) # and hasattr(pipe, "to")
pipe = pipe.to(self.device)
pipe.enable_attention_slicing()
@@ -154,7 +141,7 @@ class StableDiffusion(nn.Module):
Returns:
The loss
"""
- image = F.interpolate(image, (IMG_DIM, IMG_DIM), mode="bilinear")
+ image = F.interpolate(image, (IMG_DIM, IMG_DIM), mode="bilinear").to(torch.float16)
t = torch.randint(self.min_step, self.max_step + 1, [1], dtype=torch.long, device=self.device)
latents = self.imgs_to_latent(image)
@@ -177,9 +164,8 @@ class StableDiffusion(nn.Module):
grad = w * (noise_pred - noise)
grad = torch.nan_to_num(grad)
- if grad_scaler is not None:
- latents = grad_scaler.scale(latents)
- loss = cast(Tensor, _SDSGradient.apply(latents, grad))
+ target = (latents - grad).detach()
+ loss = 0.5 * F.mse_loss(latents, target, reduction="sum") / latents.shape[0]
return loss
@@ -230,6 +216,7 @@ class StableDiffusion(nn.Module):
# compute the previous noisy sample x_t -> x_t-1
latents = self.scheduler.step(noise_pred, t, latents)["prev_sample"] # type: ignore
+ assert isinstance(latents, Tensor)
return latents
def latents_to_img(self, latents: Float[Tensor, "BS 4 H W"]) -> Float[Tensor, "BS 3 H W"]:
diff --git a/nerfstudio/generative/utils.py b/nerfstudio/generative/utils.py
index e5996da6d..ec5a72844 100644
--- a/nerfstudio/generative/utils.py
+++ b/nerfstudio/generative/utils.py
@@ -12,28 +12,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""Utility helper functions for diffusion models"""
-import torch
-from torch.cuda.amp import custom_bwd, custom_fwd
-
-"""Gradient helper functions for SDS loss"""
+import sys
+from nerfstudio.utils.rich_utils import CONSOLE
-class _SDSGradient(torch.autograd.Function):
- """Custom gradient function for SDS loss. Since it is already computed, we can just return it."""
+class CatchMissingPackages:
+ """Class to catch missing environment packages related to diffusion models."""
- @staticmethod
- @custom_fwd
- def forward(ctx, input_tensor, gt_grad):
- del input_tensor
- ctx.save_for_backward(gt_grad)
- # Return magniture of gradient, not the actual loss.
- return torch.mean(gt_grad**2) ** 0.5
+ def __init__(self):
+ pass
- @staticmethod
- @custom_bwd
- def backward(ctx, grad):
- del grad
- (gt_grad,) = ctx.saved_tensors
- batch_size = len(gt_grad)
- return gt_grad / batch_size, None
+ def __call__(self, *args, **kwargs):
+ CONSOLE.print("[bold red]Missing Stable Diffusion packages.")
+ CONSOLE.print(r"Install using [yellow]pip install nerfstudio\[gen][/yellow]")
+ CONSOLE.print(r"or [yellow]pip install -e .\[gen][/yellow] if installing from source.")
+ sys.exit(1)
+
+ def __getattr__(self, attr):
+ return self.__call__
diff --git a/nerfstudio/model_components/scene_colliders.py b/nerfstudio/model_components/scene_colliders.py
index 3e2e793ae..69c073b3b 100644
--- a/nerfstudio/model_components/scene_colliders.py
+++ b/nerfstudio/model_components/scene_colliders.py
@@ -24,7 +24,6 @@ from torch import Tensor, nn
from nerfstudio.cameras.rays import RayBundle
from nerfstudio.data.scene_box import SceneBox
-from nerfstudio.utils.misc import torch_compile
class SceneCollider(nn.Module):
@@ -109,7 +108,6 @@ class AABBBoxCollider(SceneCollider):
return ray_bundle
-@torch_compile(dynamic=True, mode="reduce-overhead")
def _intersect_with_sphere(
rays_o: torch.Tensor, rays_d: torch.Tensor, center: torch.Tensor, radius: float = 1.0, near_plane: float = 0.0
):
diff --git a/nerfstudio/models/base_model.py b/nerfstudio/models/base_model.py
index 1feb22adb..6f31c3b30 100644
--- a/nerfstudio/models/base_model.py
+++ b/nerfstudio/models/base_model.py
@@ -50,6 +50,8 @@ class ModelConfig(InstantiateConfig):
"""parameters to instantiate density field with"""
eval_num_rays_per_chunk: int = 4096
"""specifies number of rays per chunk during eval"""
+ prompt: Optional[str] = None
+ """A prompt to be used in text to NeRF models"""
class Model(nn.Module):
diff --git a/nerfstudio/models/generfacto.py b/nerfstudio/models/generfacto.py
new file mode 100644
index 000000000..bf34eb335
--- /dev/null
+++ b/nerfstudio/models/generfacto.py
@@ -0,0 +1,528 @@
+# Copyright 2022 The Nerfstudio Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Nerfstudio's Text to 3D model.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Tuple, Type
+
+import numpy as np
+import torch
+from torch.nn import Parameter
+from typing_extensions import Literal
+
+from nerfstudio.cameras.rays import RayBundle
+from nerfstudio.engine.callbacks import (
+ TrainingCallback,
+ TrainingCallbackAttributes,
+ TrainingCallbackLocation,
+)
+from nerfstudio.field_components.field_heads import FieldHeadNames
+from nerfstudio.fields.density_fields import HashMLPDensityField
+from nerfstudio.fields.generfacto_field import GenerfactoField
+from nerfstudio.generative.stable_diffusion import StableDiffusion
+from nerfstudio.generative.deepfloyd import DeepFloyd
+from nerfstudio.generative.positional_text_embeddings import PositionalTextEmbeddings
+from nerfstudio.model_components.losses import (
+ MSELoss,
+ distortion_loss,
+ interlevel_loss,
+ orientation_loss,
+)
+from nerfstudio.model_components.ray_samplers import (
+ ProposalNetworkSampler,
+ UniformSampler,
+)
+from nerfstudio.model_components.renderers import (
+ AccumulationRenderer,
+ DepthRenderer,
+ NormalsRenderer,
+ RGBRenderer,
+)
+from nerfstudio.model_components.scene_colliders import AABBBoxCollider, SphereCollider
+from nerfstudio.model_components.shaders import LambertianShader, NormalsShader
+from nerfstudio.models.base_model import Model, ModelConfig
+from nerfstudio.utils import colormaps, colors, math, misc
+
+
+@dataclass
+class GenerfactoModelConfig(ModelConfig):
+ """Generfacto model config"""
+
+ _target: Type = field(default_factory=lambda: GenerfactoModel)
+ """target class to instantiate"""
+ prompt: str = "a high quality photo of a ripe pineapple"
+ """prompt for stable dreamfusion"""
+
+ orientation_loss_mult: Tuple[float, float] = (0.001, 10.0)
+ """Orientation loss multipier on computed normals."""
+ orientation_loss_mult_range: Tuple[int, int] = (0, 15000)
+ """number of iterations to reach last orientation_loss_mult value"""
+ random_light_source: bool = True
+ """Randomizes light source per output."""
+ initialize_density: bool = True
+ """Initialize density in center of scene."""
+ taper_range: Tuple[int, int] = (0, 1000)
+ """Range of step values for the density tapering"""
+ taper_strength: Tuple[float, float] = (1.0, 0.0)
+ """Strength schedule of center density"""
+ sphere_collider: bool = True
+ """Use spherical collider instead of box"""
+ random_background: bool = True
+ """Randomly choose between using background mlp and random color for background"""
+
+ target_transmittance_start: float = 0.4
+ """target transmittance for opacity penalty. This is the percent of the scene that is
+ background when rendered at the start of training"""
+ target_transmittance_end: float = 0.7
+ """target transmittance for opacity penalty. This is the percent of the scene that is
+ background when rendered at the end of training"""
+ transmittance_end_schedule: int = 1500
+ """number of iterations to reach target_transmittance_end"""
+
+ num_proposal_samples_per_ray: Tuple[int, ...] = (256, 96)
+ """Number of samples per ray for each proposal network."""
+ num_nerf_samples_per_ray: int = 48
+ """Number of samples per ray for the nerf network."""
+ proposal_update_every: int = 5
+ """Sample every n steps after the warmup"""
+ proposal_warmup: int = 5000
+ """Scales n from 1 to proposal_update_every over this many steps"""
+ num_proposal_iterations: int = 2
+ """Number of proposal network iterations."""
+ use_same_proposal_network: bool = False
+ """Use the same proposal network. Otherwise use different ones."""
+ proposal_net_args_list: List[Dict] = field(
+ default_factory=lambda: [
+ {"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 128},
+ {"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 256},
+ ]
+ )
+ """Arguments for the proposal density fields."""
+ proposal_weights_anneal_slope: float = 10.0
+ """Slope of the annealing function for the proposal weights."""
+ proposal_weights_anneal_max_num_iters: int = 500
+ """Max num iterations for the annealing function."""
+ use_single_jitter: bool = True
+ """Whether use single jitter or not for the proposal networks."""
+ interlevel_loss_mult: float = 1.0
+ """Proposal loss multiplier."""
+ distortion_loss_mult: float = 1.0
+ """Distortion loss multiplier."""
+ start_normals_training: int = 1000
+ """Start training normals after this many iterations"""
+ start_lambertian_training: int = 1000
+ """start training with lambertian shading after this many iterations"""
+ opacity_penalty: bool = True
+ """enables penalty to encourage sparse weights (penalizing for uniform density along ray)"""
+ opacity_loss_mult: float = 1
+ """scale for opacity penalty"""
+ max_res: int = 256
+ """Maximum resolution of the density field."""
+
+ location_based_prompting: bool = True
+ """enables location based prompting"""
+ interpolated_prompting: bool = False
+ """enables interpolated location prompting"""
+ positional_prompting: Literal["discrete", "interpolated", "off"] = "discrete"
+ """ how to incorporate position into prompt"""
+ top_prompt: str = ", overhead view"
+ """appended to prompt for overhead view"""
+ side_prompt: str = ", side view"
+ """appended to prompt for side view"""
+ front_prompt: str = ", front view"
+ """appended to prompt for front view"""
+ back_prompt: str = ", back view"
+ """appended to prompt for back view"""
+ guidance_scale: float = 20
+ """guidance scale for sds loss"""
+ diffusion_device: Optional[str] = None
+ """device for diffusion model"""
+ diffusion_model: Literal["stablediffusion", "deepfloyd"] = "deepfloyd"
+ """diffusion model for SDS loss"""
+ sd_version: str = "1-5"
+ """model version when using stable diffusion"""
+ implementation: Literal["tcnn", "torch"] = "tcnn"
+ """Which implementation to use for the model."""
+
+
+class GenerfactoModel(Model):
+ """Generfacto Model
+
+ Args:
+ config: Generfacto configuration to instantiate model
+ """
+
+ config: GenerfactoModelConfig
+
+ def __init__(
+ self,
+ config: GenerfactoModelConfig,
+ **kwargs,
+ ) -> None:
+ self.prompt = config.prompt
+ self.cur_prompt = config.prompt
+ self.sd_version = config.sd_version
+ self.initialize_density = config.initialize_density
+ self.train_normals = False
+ self.train_shaded = False
+ self.random_background = config.random_background
+ self.density_strength = 1.0
+ self.target_transmittance = config.target_transmittance_start
+ self.grad_scaler = kwargs["grad_scaler"]
+
+ self.guidance_scale = config.guidance_scale
+ self.top_prompt = config.top_prompt
+ self.side_prompt = config.side_prompt
+ self.back_prompt = config.back_prompt
+ self.front_prompt = config.front_prompt
+
+ self.diffusion_device = (
+ torch.device(kwargs["device"]) if config.diffusion_device is None else torch.device(config.diffusion_device)
+ )
+
+ super().__init__(config=config, **kwargs)
+
+ def populate_modules(self):
+ """Set the fields and modules"""
+ super().populate_modules()
+
+ if self.config.diffusion_model == "stablediffusion":
+ self._diffusion_model = StableDiffusion(self.diffusion_device, version=self.sd_version)
+ elif self.config.diffusion_model == "deepfloyd":
+ self._diffusion_model = DeepFloyd(self.diffusion_device)
+
+ self.text_embeddings = PositionalTextEmbeddings(
+ base_prompt=self.cur_prompt,
+ top_prompt=self.cur_prompt + self.top_prompt,
+ side_prompt=self.cur_prompt + self.side_prompt,
+ back_prompt=self.cur_prompt + self.back_prompt,
+ front_prompt=self.cur_prompt + self.front_prompt,
+ diffusion_model=self._diffusion_model,
+ positional_prompting=self.config.positional_prompting,
+ )
+
+ # setting up fields
+ self.field = GenerfactoField(self.scene_box.aabb, max_res=self.config.max_res)
+
+ # samplers
+ self.density_fns = []
+ num_prop_nets = self.config.num_proposal_iterations
+ # Build the proposal network(s)
+ self.proposal_networks = torch.nn.ModuleList()
+
+ for i in range(num_prop_nets):
+ prop_net_args = self.config.proposal_net_args_list[min(i, len(self.config.proposal_net_args_list) - 1)]
+ network = HashMLPDensityField(
+ self.scene_box.aabb, **prop_net_args, implementation=self.config.implementation
+ )
+ self.proposal_networks.append(network)
+ self.density_fns.extend([network.density_fn for network in self.proposal_networks])
+
+ def update_schedule(step):
+ return np.clip(
+ np.interp(step, [0, self.config.proposal_warmup], [0, self.config.proposal_update_every]),
+ 1,
+ self.config.proposal_update_every,
+ )
+
+ self.proposal_sampler = ProposalNetworkSampler(
+ num_nerf_samples_per_ray=self.config.num_nerf_samples_per_ray,
+ num_proposal_samples_per_ray=self.config.num_proposal_samples_per_ray,
+ num_proposal_network_iterations=self.config.num_proposal_iterations,
+ single_jitter=self.config.use_single_jitter,
+ update_sched=update_schedule,
+ initial_sampler=UniformSampler(single_jitter=self.config.use_single_jitter),
+ )
+
+ # renderers
+ self.renderer_rgb = RGBRenderer(background_color=colors.WHITE)
+ self.renderer_accumulation = AccumulationRenderer()
+ self.renderer_depth = DepthRenderer()
+ self.renderer_normals = NormalsRenderer()
+
+ # shaders
+ self.shader_lambertian = LambertianShader()
+ self.shader_normals = NormalsShader()
+
+ # losses
+ self.rgb_loss = MSELoss()
+
+ # colliders
+ if self.config.sphere_collider:
+ self.collider = SphereCollider(torch.Tensor([0, 0, 0]), 1.0)
+ else:
+ self.collider = AABBBoxCollider(scene_box=self.scene_box)
+
+ def get_training_callbacks(
+ self, training_callback_attributes: TrainingCallbackAttributes
+ ) -> List[TrainingCallback]:
+ # the callback that we want to run every X iterations after the training iteration
+ def taper_density(
+ self, training_callback_attributes: TrainingCallbackAttributes, step: int # pylint: disable=unused-argument
+ ):
+ self.density_strength = np.interp(step, self.config.taper_range, self.config.taper_strength)
+
+ def start_training_normals(
+ self, training_callback_attributes: TrainingCallbackAttributes, step: int # pylint: disable=unused-argument
+ ):
+ self.train_normals = True
+
+ def start_shaded_training(
+ self, training_callback_attributes: TrainingCallbackAttributes, step: int # pylint: disable=unused-argument
+ ):
+ self.train_shaded = True
+
+ def update_orientation_loss_mult(
+ self, training_callback_attributes: TrainingCallbackAttributes, step: int # pylint: disable=unused-argument
+ ):
+ if step <= self.config.start_normals_training:
+ self.orientation_loss_mult = 0
+ else:
+ self.orientation_loss_mult = np.interp(
+ step,
+ self.config.orientation_loss_mult_range,
+ self.config.orientation_loss_mult,
+ )
+
+ # anneal the weights of the proposal network before doing PDF sampling
+ def set_anneal(step):
+ # https://arxiv.org/pdf/2111.12077.pdf eq. 18
+ train_frac = np.clip(step / self.config.proposal_weights_anneal_max_num_iters, 0, 1)
+
+ def bias(x, b):
+ return b * x / ((b - 1) * x + 1)
+
+ anneal = bias(train_frac, self.config.proposal_weights_anneal_slope)
+ self.proposal_sampler.set_anneal(anneal)
+
+ callbacks = [
+ TrainingCallback(
+ where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION],
+ func=taper_density,
+ update_every_num_iters=1,
+ args=[self, training_callback_attributes],
+ ),
+ TrainingCallback(
+ where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION],
+ iters=(self.config.start_normals_training,),
+ func=start_training_normals,
+ args=[self, training_callback_attributes],
+ ),
+ TrainingCallback(
+ where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION],
+ iters=(self.config.start_lambertian_training,),
+ func=start_shaded_training,
+ args=[self, training_callback_attributes],
+ ),
+ TrainingCallback(
+ where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION],
+ update_every_num_iters=1,
+ func=update_orientation_loss_mult,
+ args=[self, training_callback_attributes],
+ ),
+ TrainingCallback(
+ where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION],
+ update_every_num_iters=1,
+ func=self.proposal_sampler.step_cb,
+ ),
+ TrainingCallback(
+ where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION],
+ update_every_num_iters=1,
+ func=set_anneal,
+ ),
+ ]
+ return callbacks
+
+ def get_param_groups(self) -> Dict[str, List[Parameter]]:
+ param_groups = {}
+ param_groups["proposal_networks"] = list(self.proposal_networks.parameters())
+ param_groups["fields"] = list(self.field.parameters())
+ return param_groups
+
+ def get_outputs(self, ray_bundle: RayBundle): # pylint: disable=too-many-statements
+ # uniform sampling
+ background_rgb = self.field.get_background_rgb(ray_bundle)
+ ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns)
+ field_outputs = self.field(ray_samples, compute_normals=True)
+ density = field_outputs[FieldHeadNames.DENSITY]
+
+ if self.initialize_density:
+ pos = ray_samples.frustums.get_positions()
+ density_blob = self.density_strength * (-torch.exp(torch.norm(pos, dim=-1) / 0.4) + 2)[..., None]
+ density = torch.max(density + density_blob, torch.tensor([0.0], device=self.device))
+
+ weights = ray_samples.get_weights(density)
+ weights_list.append(weights)
+ ray_samples_list.append(ray_samples)
+
+ accumulation = self.renderer_accumulation(weights)
+ depth = self.renderer_depth(weights, ray_samples)
+ rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights)
+
+ accum_mask = torch.clamp((torch.nan_to_num(accumulation, nan=0.0)), min=0.0, max=1.0)
+ accum_mask_inv = 1.0 - accum_mask
+
+ background = accum_mask_inv * background_rgb
+
+ outputs = {
+ "rgb_only": rgb,
+ "background_rgb": background_rgb,
+ "background": background,
+ "accumulation": accum_mask,
+ "depth": depth,
+ }
+
+ # These use a lot of GPU memory, so we avoid storing them for eval.
+ if self.training:
+ outputs["weights_list"] = weights_list
+ outputs["ray_samples_list"] = ray_samples_list
+
+ for i in range(self.config.num_proposal_iterations):
+ outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i])
+
+ normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights)
+
+ # lambertian shading
+ if self.config.random_light_source: # and self.training:
+ light_d = ray_bundle.origins[0] + torch.randn(3, dtype=torch.float).to(normals)
+ else:
+ light_d = ray_bundle.origins[0]
+ light_d = math.safe_normalize(light_d)
+
+ if (self.train_shaded and np.random.random_sample() > 0.75) or not self.training:
+ shading_weight = 0.9
+ else:
+ shading_weight = 0.0
+
+ shaded, shaded_albedo = self.shader_lambertian(
+ rgb=rgb, normals=normals, light_direction=light_d, shading_weight=shading_weight, detach_normals=False
+ )
+
+ shaded, shaded_albedo = accum_mask * shaded, accum_mask * shaded_albedo
+
+ outputs["normals"] = self.shader_normals(normals, weights=accum_mask)
+ outputs["shaded"] = shaded
+ outputs["other_train_output"] = shaded_albedo + background
+ outputs["shaded_albedo"] = shaded_albedo
+ outputs["rgb"] = accum_mask * rgb + background
+
+ # while training 50% of the time use a random background
+ if np.random.random_sample() < 0.5 and self.random_background and self.training:
+ background = torch.ones_like(background) * torch.rand(3, device=self.device) * accum_mask_inv
+
+ if shading_weight > 0:
+ samp = np.random.random_sample()
+ if samp > 0.5:
+ outputs["train_output"] = outputs["shaded"]
+ else:
+ outputs["train_output"] = shaded_albedo + background
+ else:
+ outputs["train_output"] = accum_mask * rgb + background
+
+ outputs["rendered_orientation_loss"] = orientation_loss(
+ weights.detach(), field_outputs[FieldHeadNames.NORMALS], ray_bundle.directions
+ )
+
+ assert weights.shape[-1] == 1
+ if self.config.opacity_penalty:
+ outputs["opacity_loss"] = torch.sqrt(torch.sum(weights, dim=-2) ** 2 + 0.01) * self.config.opacity_loss_mult
+
+ return outputs
+
+ def get_loss_dict(self, outputs, batch, metrics_dict=None) -> Dict[str, torch.Tensor]:
+ # Scaling metrics by coefficients to create the losses.
+
+ loss_dict = {}
+ loss_dict = misc.scale_dict(loss_dict, self.config.loss_coefficients)
+ if self.train_normals:
+ # orientation loss for computed normals
+ loss_dict["orientation_loss"] = self.orientation_loss_mult * torch.mean(
+ outputs["rendered_orientation_loss"]
+ )
+ else:
+ loss_dict["orientation_loss"] = 0
+
+ if self.config.opacity_penalty:
+ loss_dict["opacity_loss"] = self.config.opacity_loss_mult * outputs["opacity_loss"].mean()
+
+ if self.prompt != self.cur_prompt:
+ self.cur_prompt = self.prompt
+ self.text_embeddings.update_prompt(
+ base_prompt=self.cur_prompt,
+ top_prompt=self.cur_prompt + self.top_prompt,
+ side_prompt=self.cur_prompt + self.side_prompt,
+ back_prompt=self.cur_prompt + self.back_prompt,
+ front_prompt=self.cur_prompt + self.front_prompt,
+ )
+
+ text_embedding = self.text_embeddings.get_text_embedding(
+ vertical_angle=batch["vertical"], horizontal_angle=batch["central"]
+ )
+
+ train_output = (
+ outputs["train_output"]
+ .view(1, int(outputs["train_output"].shape[0] ** 0.5), int(outputs["train_output"].shape[0] ** 0.5), 3)
+ .permute(0, 3, 1, 2)
+ )
+
+ sds_loss = self._diffusion_model.sds_loss(
+ text_embedding.to(self.diffusion_device),
+ train_output.to(self.diffusion_device),
+ guidance_scale=int(self.guidance_scale),
+ grad_scaler=self.grad_scaler,
+ )
+
+ loss_dict["sds_loss"] = sds_loss.to(self.device)
+
+ if self.training:
+ loss_dict["distortion_loss"] = self.config.distortion_loss_mult * distortion_loss(
+ outputs["weights_list"], outputs["ray_samples_list"]
+ )
+ loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss(
+ outputs["weights_list"], outputs["ray_samples_list"]
+ )
+ return loss_dict
+
+ def get_image_metrics_and_images(
+ self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor]
+ ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]:
+ acc = colormaps.apply_colormap(outputs["accumulation"])
+ depth = colormaps.apply_depth_colormap(
+ outputs["depth"],
+ accumulation=outputs["accumulation"],
+ )
+ prop_depth_0 = colormaps.apply_depth_colormap(
+ outputs["prop_depth_0"],
+ accumulation=outputs["accumulation"],
+ )
+ prop_depth_1 = colormaps.apply_depth_colormap(
+ outputs["prop_depth_1"],
+ accumulation=outputs["accumulation"],
+ )
+
+ metrics_dict = {}
+ images_dict = {
+ "img": outputs["rgb"],
+ "accumulation": acc,
+ "depth": depth,
+ "prop_depth_0": prop_depth_0,
+ "prop_depth_1": prop_depth_1,
+ "normals": outputs["normals"],
+ }
+ return metrics_dict, images_dict
diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py
index 1e1e0fab5..e9d30a9ab 100644
--- a/nerfstudio/pipelines/base_pipeline.py
+++ b/nerfstudio/pipelines/base_pipeline.py
@@ -40,8 +40,8 @@ from torch.cuda.amp.grad_scaler import GradScaler
from nerfstudio.configs import base_config as cfg
from nerfstudio.data.datamanagers.base_datamanager import (
DataManager,
+ DataManagerConfig,
VanillaDataManager,
- VanillaDataManagerConfig,
)
from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes
from nerfstudio.models.base_model import Model, ModelConfig
@@ -205,7 +205,7 @@ class VanillaPipelineConfig(cfg.InstantiateConfig):
_target: Type = field(default_factory=lambda: VanillaPipeline)
"""target class to instantiate"""
- datamanager: VanillaDataManagerConfig = VanillaDataManagerConfig()
+ datamanager: DataManagerConfig = DataManagerConfig()
"""specifies the datamanager config"""
model: ModelConfig = ModelConfig()
"""specifies the model config"""
@@ -242,7 +242,7 @@ class VanillaPipeline(Pipeline):
super().__init__()
self.config = config
self.test_mode = test_mode
- self.datamanager: VanillaDataManager = config.datamanager.setup(
+ self.datamanager: DataManager = config.datamanager.setup(
device=device, test_mode=test_mode, world_size=world_size, local_rank=local_rank
)
self.datamanager.to(device)
@@ -347,6 +347,7 @@ class VanillaPipeline(Pipeline):
"""
self.eval()
metrics_dict_list = []
+ assert isinstance(self.datamanager, VanillaDataManager)
num_images = len(self.datamanager.fixed_indices_eval_dataloader)
with Progress(
TextColumn("[progress.description]{task.description}"),
diff --git a/nerfstudio/scripts/train.py b/nerfstudio/scripts/train.py
index cd5e73324..9ca18c71a 100644
--- a/nerfstudio/scripts/train.py
+++ b/nerfstudio/scripts/train.py
@@ -231,6 +231,10 @@ def main(config: TrainerConfig) -> None:
CONSOLE.log("Using --data alias for --data.pipeline.datamanager.data")
config.pipeline.datamanager.data = config.data
+ if config.prompt:
+ CONSOLE.log("Using --prompt alias for --data.pipeline.model.prompt")
+ config.pipeline.model.prompt = config.prompt
+
if config.load_config:
CONSOLE.log(f"Loading pre-set config from: {config.load_config}")
config = yaml.load(config.load_config.read_text(), Loader=yaml.Loader)
diff --git a/nerfstudio/utils/eval_utils.py b/nerfstudio/utils/eval_utils.py
index acd8bee18..13948678b 100644
--- a/nerfstudio/utils/eval_utils.py
+++ b/nerfstudio/utils/eval_utils.py
@@ -89,7 +89,6 @@ def eval_setup(
assert isinstance(config, TrainerConfig)
config.pipeline.datamanager._target = all_methods[config.method_name].pipeline.datamanager._target
-
if eval_num_rays_per_chunk:
config.pipeline.model.eval_num_rays_per_chunk = eval_num_rays_per_chunk
diff --git a/pyproject.toml b/pyproject.toml
index 02972b0e9..b3452febe 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -68,6 +68,9 @@ dependencies = [
gen = [
"diffusers==0.16.1",
"transformers==4.29.2",
+ "accelerate==0.19.0",
+ "bitsandbytes==0.39.0",
+ "sentencepiece==0.1.99",
]
diff --git a/tests/pipelines/test_vanilla_pipeline.py b/tests/pipelines/test_vanilla_pipeline.py
index 2218294b7..c27aa2069 100644
--- a/tests/pipelines/test_vanilla_pipeline.py
+++ b/tests/pipelines/test_vanilla_pipeline.py
@@ -11,10 +11,10 @@ from nerfstudio.data.datasets.base_dataset import DataparserOutputs, InputDatase
from nerfstudio.pipelines.base_pipeline import (
Model,
ModelConfig,
- VanillaDataManagerConfig,
VanillaPipeline,
VanillaPipelineConfig,
)
+from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManagerConfig
class MockedDataManager:
diff --git a/tests/test_train.py b/tests/test_train.py
index a5749c022..d5d25c141 100644
--- a/tests/test_train.py
+++ b/tests/test_train.py
@@ -25,6 +25,7 @@ BLACKLIST = [
"phototourism",
"depth-nerfacto",
"neus",
+ "generfacto",
"neus-facto",
]