Merge branch 'generative' of https://github.com/plenoptix/nerfactory into terrance/single_img_generative

This commit is contained in:
terrance committed 2023-02-21 20:23:14 +00:00
commit 0e4dcb73c3
42 files changed
+877 -383

No files matched your search

+13
View File
@@ -0,0 +1,13 @@
{
"name": "NerfStudio Dev Container",
"dockerComposeFile": "docker-compose.yml",
"service": "nerfstudio",
"workspaceFolder": "/workspace",
"shutdownAction": "stopCompose",
"extensions": [
"mutantdino.resourcemonitor",
"ms-azuretools.vscode-docker",
"nvidia.nsight-vscode-edition",
"ms-python.python"
]
}
+28
View File
@@ -0,0 +1,28 @@
version: "3.8"
services:
nerfstudio:
shm_size: '12gb'
image: dromni/nerfstudio:0.1.17
# build:
# context: ..
# dockerfile: ./Dockerfile
stdin_open: true
tty: true
environment:
NVIDIA_DRIVER_CAPABILITIES: compute,utility,graphics
DISPLAY: $DISPLAY
ports:
- "7007:7007"
volumes:
- ../:/workspace/
- /tmp/.X11-unix:/tmp/.X11-unix
working_dir: /workspace/
command: /bin/bash
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: 1
capabilities: [ gpu ]
+27 -13
View File
@@ -1,14 +1,16 @@
# Define base image.
FROM nvidia/cuda:11.7.1-devel-ubuntu22.04
FROM nvidia/cuda:11.8.0-devel-ubuntu22.04
# Variables used at build time.
## CUDA architectures, required by Colmap and tiny-cuda-nn.
## NOTE: All commonly used GPU architectures are included and supported here. To speedup the image build process remove all architectures but the one of your explicit GPU. Find details here: https://developer.nvidia.com/cuda-gpus (8.6 translates to 86 in the line below) or in the docs.
ARG CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37
# Set environment variables.
## Set non-interactive to prevent asking for user inputs blocking image creation.
ENV DEBIAN_FRONTEND=noninteractive
## Set timezone as it is required by some packages.
ENV TZ=Europe/Berlin
## CUDA architectures, required by tiny-cuda-nn.
## NOTE: All commonly used GPU architectures are included and supported here. To speedup the image build process remove all architectures but the one of your explicit GPU. Find details here: https://developer.nvidia.com/cuda-gpus (8.6 translates to 86 in the line below) or in the docs.
ENV TCNN_CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37
## CUDA Home, required to find CUDA in some packages.
ENV CUDA_HOME="/usr/local/cuda"
@@ -17,6 +19,7 @@ RUN apt-get update && \
apt-get install -y --no-install-recommends \
build-essential \
cmake \
curl \
ffmpeg \
git \
libatlas-base-dev \
@@ -27,6 +30,7 @@ RUN apt-get update && \
libboost-test-dev \
libcgal-dev \
libeigen3-dev \
libflann-dev \
libfreeimage-dev \
libgflags-dev \
libglew-dev \
@@ -34,15 +38,18 @@ RUN apt-get update && \
libmetis-dev \
libprotobuf-dev \
libqt5opengl5-dev \
libsqlite3-dev \
libsuitesparse-dev \
nano \
protobuf-compiler \
python3.10-dev \
python3-pip \
qtbase5-dev \
sudo \
wget && \
rm -rf /var/lib/apt/lists/*
# Install GLOG (required by ceres).
RUN git clone --branch v0.6.0 https://github.com/google/glog.git --single-branch && \
cd glog && \
@@ -69,22 +76,28 @@ RUN git clone --branch 2.1.0 https://ceres-solver.googlesource.com/ceres-solver.
rm -rf ceres-solver
# Install colmap.
RUN git clone --branch 3.7 https://github.com/colmap/colmap.git --single-branch && \
RUN git clone --branch 3.8 https://github.com/colmap/colmap.git --single-branch && \
cd colmap && \
mkdir build && \
cd build && \
cmake .. -DCUDA_ENABLED=ON \
-DCUDA_NVCC_FLAGS="--std c++14" && \
-DCUDA_NVCC_FLAGS="--std c++14" \
-DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES} && \
make -j && \
make install && \
cd ../.. && \
rm -rf colmap
# Create non root user and setup environment.
RUN useradd -m -d /home/user -u 1000 user
RUN useradd -m -d /home/user -g root -G sudo -u 1000 user
RUN usermod -aG sudo user
# Set user password
RUN echo "user:user" | chpasswd
# Ensure sudo group users are not asked for a password when using sudo command by ammending sudoers file
RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
# Switch to new uer and workdir.
USER 1000:1000
USER 1000
WORKDIR /home/user
# Add local user binary folder to PATH variable.
@@ -93,16 +106,17 @@ SHELL ["/bin/bash", "-c"]
# Upgrade pip and install packages.
RUN python3.10 -m pip install --upgrade pip setuptools pathtools promise
# Install pytorch and submodules.
RUN python3.10 -m pip install torch==1.12.1+cu116 torchvision==0.13.1+cu116 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu116
# Install tynyCUDNN.
# Install pytorch and submodules (Currently, we still use cu116 which is the latest version for toch 1.12.1 and is compatible with CUDA 11.8).
RUN python3.10 -m pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116
# Install tynyCUDNN (we need to set the target architectures as environment variable first).
ENV TCNN_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES}
RUN python3.10 -m pip install git+https://github.com/NVlabs/tiny-cuda-nn.git#subdirectory=bindings/torch
# Copy nerfstudio folder and give ownership to user.
ADD . /home/user/nerfstudio
USER root
RUN chown -R user:user /home/user/nerfstudio
USER 1000:1000
RUN chown -R user /home/user/nerfstudio
USER 1000
# Install nerfstudio dependencies.
RUN cd nerfstudio && \
+2
View File
@@ -302,6 +302,8 @@ We provide the following support structures to make life easier for getting star
# Citation
You can find a paper writeup of the framework on [arXiv](https://arxiv.org/abs/2302.04264).
If you use this library or find the documentation useful for your research, please consider citing:
```
+2
View File
@@ -155,6 +155,8 @@ We'll be constantly growing this list! So make sure to check back in to see our
## Citation
You can find a paper writeup of the framework on [arXiv](https://arxiv.org/abs/2302.04264).
If you use this library or find the documentation useful for your research, please consider citing:
```none
+8 -6
View File
@@ -92,8 +92,8 @@ pip install -e .[docs]
## Use docker image
Instead of installing and compiling prerequisites, setting up the environment and installing dependencies, a ready to use docker image is provided.
### Prerequisites
Docker ([get docker](https://docs.docker.com/get-docker/)) and nvidia GPU drivers ([get nvidia drivers](https://www.nvidia.de/Download/index.aspx?lang=de)), capable of working with CUDA 11.7, must be installed.
The docker image can then either be pulled from [here](https://hub.docker.com/r/dromni/nerfstudio/tags) (replace <version> with the actual version, e.g. 0.1.10)
Docker ([get docker](https://docs.docker.com/get-docker/)) and nvidia GPU drivers ([get nvidia drivers](https://www.nvidia.de/Download/index.aspx?lang=de)), capable of working with CUDA 11.8, must be installed.
The docker image can then either be pulled from [here](https://hub.docker.com/r/dromni/nerfstudio/tags) (replace <version> with the actual version, e.g. 0.1.17)
```bash
docker pull dromni/nerfstudio:<version>
```
@@ -110,13 +110,15 @@ docker run --gpus all \ # Give the conta
-p 7007:7007 \ # Map port from local machine to docker container (required to access the web interface/UI).
--rm \ # Remove container after it is closed (recommended).
-it \ # Start container in interactive mode.
nerfstudio # Docker image name
dromni/nerfstudio:<tag> # Docker image name if you pulled from docker hub.
<--- OR --->
nerfstudio # Docker image tag if you built the image from the Dockerfile by yourself using the command from above.
```
### Call nerfstudio commands directly
Besides, the container can also directly be used by adding the nerfstudio command to the end.
```bash
docker run --gpus all -v /folder/of/your/data:/workspace/ -v /home/<YOUR_USER>/.cache/:/home/user/.cache/ -p 7007:7007 --rm -it # Parameters.
nerfstudio \ # Docker image name
dromni/nerfstudio:<tag> \ # Docker image name
ns-process-data video --data /workspace/video.mp4 # Smaple command of nerfstudio.
```
### Note
@@ -125,10 +127,10 @@ docker run --gpus all -v /folder/of/your/data:/workspace/ -v /home/<YOUR_USER>/.
- Always use full paths, relative paths are known to create issues when being used in mounts into docker.
- Everything inside the container, what is not in a mounted folder (workspace in the above example), will be permanently removed after destroying the container. Always do all your tasks and output folder in workdir!
- The user inside the container is called 'user' and is mapped to the local user with ID 1000 (usually the first non-root user on Linux systems).
- The container currently is based on nvidia/cuda:11.7.1-devel-ubuntu22.04, consequently it comes with CUDA 11.7 which must be supported by the nvidia driver. No local CUDA installation is required or will be affected by using the docker image.
- The container currently is based on nvidia/cuda:11.8.0-devel-ubuntu22.04, consequently it comes with CUDA 11.8 which must be supported by the nvidia driver. No local CUDA installation is required or will be affected by using the docker image.
- The docker image (respectively Ubuntu 22.04) comes with Python3.10, no older version of Python is installed.
- If you call the container with commands directly, you still might want to add the interactive terminal ('-it') flag to get live log outputs of the nerfstudio scripts. In case the container is used in an automated environment the flag should be discarded.
- The current version of docker is built for multi-architecture (CUDA architectures) use. The target architecture must be defined at build time for tinyCUDNN to be able to compile properly. If your GPU architecture is not covered by the following table you need to replace the number in the line ```ENV TCNN_CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37``` to you specific architecture. It also is a good idea to remove all architectures but yours (e.g. ```ENV TCNN_CUDA_ARCHITECTURES=86```) to speedup the docker build a lot.
- The current version of docker is built for multi-architecture (CUDA architectures) use. The target architecture(s) must be defined at build time for Colmap and tinyCUDNN to be able to compile properly. If your GPU architecture is not covered by the following table you need to replace the number in the line ```ARG CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37``` to your specific architecture. It also is a good idea to remove all architectures but yours (e.g. ```ARG CUDA_ARCHITECTURES=86```) to speedup the docker build process a lot.
**Currently supported CUDA architectures in the docker image**
+2
View File
@@ -12,6 +12,7 @@ Here are the popular commands that we offer. If you've cloned the repo, you can
| ns-process-data | Generate a dataset from your own data | scripts/process_data.py |
| ns-download-data | Download existing captures | scripts/downloads/download_data.py |
| ns-train | Generate a NeRF | scripts/train.py |
| ns-viewer | View a trained NeRF | scripts/viewer/run_viewer.py |
| ns-eval | Run evaluation metrics for your Model | scripts/eval.py |
| ns-render | Render out a video of your NeRF | scripts/render.py |
| ns-export | Export a NeRF into other formats | scripts/exporter.py |
@@ -24,6 +25,7 @@ ns_process_data
ns_download_data
ns_train
ns_render
ns_viewer
ns_export
ns_eval
```
+9
View File
@@ -0,0 +1,9 @@
# ns-viewer
```{eval-rst}
.. argparse::
:module: scripts.viewer.run_viewer
:func: get_parser_fn
:prog: ns-viewer
:nodefault:
```
+1 -1
View File
@@ -367,7 +367,7 @@ method_configs["dreamfusion"] = TrainerConfig(
start_lambertian_training=1000,
start_normals_training=500,
opacity_loss_mult=0.001,
prompting_type="location_based",
positional_prompting="discrete",
guidance_scale=100,
),
),
@@ -51,7 +51,11 @@ from nerfstudio.data.dataparsers.phototourism_dataparser import (
)
from nerfstudio.data.dataparsers.sdfstudio_dataparser import SDFStudioDataParserConfig
from nerfstudio.data.datasets.base_dataset import InputDataset
from nerfstudio.data.pixel_samplers import EquirectangularPixelSampler, PixelSampler
from nerfstudio.data.pixel_samplers import (
EquirectangularPixelSampler,
PatchPixelSampler,
PixelSampler,
)
from nerfstudio.data.utils.dataloaders import (
CacheDataloader,
FixedIndicesEvalDataloader,
@@ -283,6 +287,8 @@ class VanillaDataManagerConfig(InstantiateConfig):
"""The scale factor for scaling spatial data such as images, mask, semantics
along with relevant information about camera intrinsics
"""
patch_size: int = 1
"""Size of patch to sample from. If >1, patch-based sampling will be used."""
class VanillaDataManager(DataManager): # pylint: disable=abstract-method
@@ -346,6 +352,9 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method
self, dataset: InputDataset, *args: Any, **kwargs: Any
) -> PixelSampler:
"""Infer pixel sampler to use."""
if self.config.patch_size > 1:
return PatchPixelSampler(*args, **kwargs, patch_size=self.config.patch_size)
# If all images are equirectangular, use equirectangular pixel sampler
is_equirectangular = dataset.cameras.camera_type == CameraType.EQUIRECTANGULAR.value
if is_equirectangular.all():
+51
View File
@@ -243,3 +243,54 @@ class EquirectangularPixelSampler(PixelSampler): # pylint: disable=too-few-publ
).long()
return indices
class PatchPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods
"""Samples 'pixel_batch's from 'image_batch's. Samples square patches
from the images randomly. Useful for patch-based losses.
Args:
num_rays_per_batch: number of rays to sample per batch
keep_full_image: whether or not to include a reference to the full image in returned batch
patch_size: side length of patch. This must be consistent in the method
config in order for samples to be reshaped into patches correctly.
"""
def __init__(self, num_rays_per_batch: int, keep_full_image: bool = False, **kwargs) -> None:
self.patch_size = kwargs["patch_size"]
num_rays = (num_rays_per_batch // (self.patch_size**2)) * (self.patch_size**2)
super().__init__(num_rays, keep_full_image, **kwargs)
# overrides base method
def sample_method( # pylint: disable=no-self-use
self,
batch_size: int,
num_images: int,
image_height: int,
image_width: int,
mask: Optional[TensorType] = None,
device: Union[torch.device, str] = "cpu",
) -> TensorType["batch_size", 3]:
if mask:
# Note: if there is a mask, sampling reduces back to uniform sampling
indices = super().sample_method(batch_size, num_images, image_height, image_width, mask=mask, device=device)
else:
sub_bs = batch_size // (self.patch_size**2)
indices = torch.rand((sub_bs, 3), device=device) * torch.tensor(
[num_images, image_height - self.patch_size, image_width - self.patch_size],
device=device,
)
indices = indices.view(sub_bs, 1, 1, 3).broadcast_to(sub_bs, self.patch_size, self.patch_size, 3).clone()
yys, xxs = torch.meshgrid(
torch.arange(self.patch_size, device=device), torch.arange(self.patch_size, device=device)
)
indices[:, ..., 1] += yys
indices[:, ..., 2] += xxs
indices = torch.floor(indices).long()
indices = indices.flatten(0, 2)
return indices
+5 -1
View File
@@ -146,7 +146,11 @@ class Trainer:
'inference': does not load any dataset into memory
"""
self.pipeline = self.config.pipeline.setup(
device=self.device, test_mode=test_mode, world_size=self.world_size, local_rank=self.local_rank, grad_scaler=self.grad_scaler
device=self.device,
test_mode=test_mode,
world_size=self.world_size,
local_rank=self.local_rank,
grad_scaler=self.grad_scaler,
)
self.optimizers = self.setup_optimizers()
+4
View File
@@ -151,6 +151,10 @@ def generate_point_cloud(
CONSOLE.print(f"Please set --normal_output_name to one of: {outputs.keys()}", justify="center")
sys.exit(1)
normal = outputs[normal_output_name]
assert (
torch.min(normal) >= 0.0 and torch.max(normal) <= 1.0
), "Normal values from method output must be in [0, 1]"
normal = (normal * 2.0) - 1.0
point = ray_bundle.origins + ray_bundle.directions * depth
if use_bounding_box:
-1
View File
@@ -29,7 +29,6 @@ from nerfstudio.data.scene_box import SceneBox
from nerfstudio.field_components.activations import trunc_exp
from nerfstudio.field_components.field_heads import FieldHeadNames, PredNormalsFieldHead
from nerfstudio.fields.base_field import Field
from nerfstudio.utils import math
try:
import tinycudann as tcnn
+43 -35
View File
@@ -33,27 +33,25 @@ from torch.cuda.amp import custom_bwd, custom_fwd
from torch.cuda.amp.grad_scaler import GradScaler
from torchtyping import TensorType
CONSOLE = Console(width=120)
try:
from diffusers import PNDMScheduler, StableDiffusionPipeline
from transformers import logging
except ImportError:
CONSOLE.print("[bold red]Missing Stable Diffusion packages.")
CONSOLE.print(r"Install using [yellow]pip install nerfstudio\[gen][/yellow]")
CONSOLE.print(r"or [yellow]pip install -e .\[gen][/yellow] if installing from source.")
sys.exit(1)
SD_AVAILABLE = True
logging.set_verbosity_error()
except ImportError:
SD_AVAILABLE = False
CONSOLE = Console(width=120)
IMG_DIM = 512
CONST_SCALE = 0.18215
SD_IDENTIFIERS = {
'1-5':'runwayml/stable-diffusion-v1-5',
'2-0':'stabilityai/stable-diffusion-2-base',
'2-1':'stabilityai/stable-diffusion-2-1-base'
"1-5": "runwayml/stable-diffusion-v1-5",
"2-0": "stabilityai/stable-diffusion-2-base",
"2-1": "stabilityai/stable-diffusion-2-1-base",
}
@dataclass
class UNet2DConditionOutput:
"""Class to hold traced model"""
@@ -88,9 +86,17 @@ class StableDiffusion(nn.Module):
num_train_timesteps: number of training timesteps
"""
def __init__(self, device: Union[torch.device, str], num_train_timesteps: int = 1000, version='1-5') -> None:
def __init__(self, device: Union[torch.device, str], num_train_timesteps: int = 1000, version="1-5") -> None:
super().__init__()
if not SD_AVAILABLE:
CONSOLE.print("[bold red]Missing Stable Diffusion packages.")
CONSOLE.print(r"Install using [yellow]pip install nerfstudio\[gen][/yellow]")
CONSOLE.print(r"or [yellow]pip install -e .\[gen][/yellow] if installing from source.")
sys.exit(1)
logging.set_verbosity_error()
self.device = device
self.num_train_timesteps = num_train_timesteps
@@ -113,8 +119,8 @@ class StableDiffusion(nn.Module):
pipe.enable_attention_slicing()
# use jitted unet
filename_sd_id = sd_id.split('/')[-1]
unet_traced_filename = Path(appdirs.user_data_dir("nerfstudio")) / f'{filename_sd_id}_unet_traced.pt'
filename_sd_id = sd_id.split("/")[-1]
unet_traced_filename = Path(appdirs.user_data_dir("nerfstudio")) / f"{filename_sd_id}_unet_traced.pt"
if unet_traced_filename.exists():
CONSOLE.print("Loading traced UNet.")
unet_traced = torch.jit.load(unet_traced_filename)
@@ -198,32 +204,34 @@ class StableDiffusion(nn.Module):
Returns:
The loss
"""
image = F.interpolate(image, (IMG_DIM, IMG_DIM), mode="bilinear")
t = torch.randint(self.min_step, self.max_step + 1, [1], dtype=torch.long, device=self.device)
latents = self.imgs_to_latent(image)
# Disable autocast when using diffusers pipeline.
with torch.autocast(device_type="cuda", enabled=False):
image = F.interpolate(image.half(), (IMG_DIM, IMG_DIM), mode="bilinear")
t = torch.randint(self.min_step, self.max_step + 1, [1], dtype=torch.long, device=self.device)
latents = self.imgs_to_latent(image)
# predict the noise residual with unet, NO grad!
with torch.no_grad():
# add noise
noise = torch.randn_like(latents)
latents_noisy = self.scheduler.add_noise(latents, noise, t) # type: ignore
# pred noise
latent_model_input = torch.cat([latents_noisy] * 2)
noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings).sample
# predict the noise residual with unet, NO grad!
with torch.no_grad():
# add noise
noise = torch.randn_like(latents)
latents_noisy = self.scheduler.add_noise(latents, noise, t) # type: ignore
# pred noise
latent_model_input = torch.cat([latents_noisy] * 2)
noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings).sample
# perform guidance
noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
noise_pred = noise_pred_text + guidance_scale * (noise_pred_text - noise_pred_uncond)
# perform guidance
noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
noise_pred = noise_pred_text + guidance_scale * (noise_pred_text - noise_pred_uncond)
# w(t), sigma_t^2
w = 1 - self.alphas[t]
# w(t), sigma_t^2
w = 1 - self.alphas[t]
grad = w * (noise_pred - noise)
grad = torch.nan_to_num(grad)
grad = w * (noise_pred - noise)
grad = torch.nan_to_num(grad)
if grad_scaler is not None:
latents = grad_scaler.scale(latents)
loss = _SDSGradient.apply(latents, grad)
if grad_scaler is not None:
latents = grad_scaler.scale(latents)
loss = _SDSGradient.apply(latents, grad)
return loss
+85 -36
View File
@@ -16,46 +16,95 @@
Stable diffusion utils
"""
def get_text_embedding(batch, prompting_type, text_embeddings, sd_device):
""""""
from torchtyping import TensorType
from typing_extensions import Literal
base = text_embeddings["base_text_embedding"]
front = text_embeddings["front_text_embedding"]
side = text_embeddings["side_text_embedding"]
top = text_embeddings["top_text_embedding"]
back = text_embeddings["back_text_embedding"]
from nerfstudio.generative.stable_diffusion import StableDiffusion
assert prompting_type in ("location_based", "interpolated", "base")
if prompting_type == "location_based":
if batch["vertical"] < 40:
text_embedding = top
elif batch["central"] > 315 or batch["central"] <= 45:
text_embedding = front
elif batch["central"] > 45 and batch["central"] <= 135:
text_embedding = side
elif batch["central"] > 135 and batch["central"] <= 225:
text_embedding = back
else: # batch["central"] > 225 and batch["central"] <= 315:
text_embedding = side
class PositionalTextEmbeddings:
"""Postional Prompts. Currently the following location based text embeddings are supported:
- discrete: Choose the embedding closest to the camera position
- interpolated: Interpolate between the embeddings based on the camera position
- off: Don't modify the text embedding based on the camera position
elif prompting_type == "interpolated":
horiz = batch["central"].to(sd_device)
vert = max(batch["vertical"].to(sd_device), 0)
Args:
base_prompt: Prompt for base view
top_prompt: Prompt for top view
side_prompt: Prompt for side view
back_prompt: Prompt for back view
front_prompt: Prompt for front view
stable_diffusion: Instance of StableDiffusion
positional_prompting: how to incorporate position into prompt.
"""
if batch["central"] > 0 or batch["central"] <= 90:
text_embedding = (horiz) * side + (90 - horiz) * front
elif batch["central"] > 90 and batch["central"] <= 180:
text_embedding = (horiz - 90) * back + (180 - horiz) * side
elif batch["central"] > 180 and batch["central"] <= 270:
text_embedding = (horiz - 180) * side + (270 - horiz) * back
else: # batch["central"] > 270 and batch["central"] <= 360:
text_embedding = (horiz - 270) * front + (360 - horiz) * side
text_embedding = text_embedding / 90.0
text_embedding = (vert * text_embedding + (90 - vert) * top) / 90.0
def __init__(
self,
base_prompt: str,
top_prompt: str,
side_prompt: str,
back_prompt: str,
front_prompt: str,
stable_diffusion: StableDiffusion,
positional_prompting: Literal["discrete", "interpolated", "off"] = "discrete",
):
self.positional_prompting = positional_prompting
self.sd_device = stable_diffusion.device
self.sd = stable_diffusion
self.update_prompt(base_prompt, top_prompt, side_prompt, back_prompt, front_prompt)
elif prompting_type == "base":
text_embedding = base
def update_prompt(self, base_prompt: str, top_prompt: str, side_prompt: str, back_prompt: str, front_prompt: str):
"""Update the text embeddings based on the new prompts.
return text_embedding
Args:
base_prompt: Prompt for base view
top_prompt: Prompt for top view
side_prompt: Prompt for side view
back_prompt: Prompt for back view
front_prompt: Prompt for front view
"""
self.base_embed = self.sd.get_text_embeds(base_prompt, "")
self.top_embed = self.sd.get_text_embeds(top_prompt, "")
self.side_embed = self.sd.get_text_embeds(side_prompt, "")
self.back_embed = self.sd.get_text_embeds(back_prompt, "")
self.front_embed = self.sd.get_text_embeds(front_prompt, "")
def get_text_embedding(self, vertical_angle: TensorType[1], horizonal_angle: TensorType[1]):
"""Get text embedding based on the position of the camera relative to the scene.
This trick is used in Dreamfusion (https://dreamfusion3d.github.io/).
Args:
vertical_angle: vertical angle of the camera
horizonal_angle: horizonal angle of the camera
"""
if self.positional_prompting == "discrete":
if vertical_angle < 40:
text_embedding = self.top_embed
elif 315 < horizonal_angle <= 45:
text_embedding = self.front_embed
elif 45 < horizonal_angle <= 135:
text_embedding = self.side_embed
elif 135 < horizonal_angle <= 225:
text_embedding = self.back_embed
else: # horizonal_angle > 225 and horizonal_angle <= 315:
text_embedding = self.side_embed
elif self.positional_prompting == "interpolated":
horiz = horizonal_angle.to(self.sd_device)
vert = max(vertical_angle.to(self.sd_device), 0)
if 0 < horizonal_angle <= 90:
text_embedding = (horiz) * self.side_embed + (90 - horiz) * self.front_embed
elif 90 < horizonal_angle <= 180:
text_embedding = (horiz - 90) * self.back_embed + (180 - horiz) * self.side_embed
elif 180 < horizonal_angle <= 270:
text_embedding = (horiz - 180) * self.side_embed + (270 - horiz) * self.back_embed
else: # 270 < horizonal_angle <= 360:
text_embedding = (horiz - 270) * self.front_embed + (360 - horiz) * self.side_embed
text_embedding = text_embedding / 90.0
text_embedding = (vert * text_embedding + (90 - vert) * self.top_embed) / 90.0
else:
text_embedding = self.base_embed
return text_embedding
+217
View File
@@ -20,8 +20,10 @@ from enum import Enum
import torch
from torch import nn
from torchtyping import TensorType
from typing_extensions import Literal
from nerfstudio.cameras.rays import RaySamples
from nerfstudio.utils.math import masked_reduction
L1Loss = nn.L1Loss
MSELoss = nn.MSELoss
@@ -308,3 +310,218 @@ def depth_loss(
return urban_radiance_field_depth_loss(weights, termination_depth, predicted_depth, steps, sigma)
raise NotImplementedError("Provided depth loss type not implemented.")
def monosdf_normal_loss(
normal_pred: TensorType["num_samples", 3], normal_gt: TensorType["num_samples", 3]
) -> TensorType[0]:
"""
Normal consistency loss proposed in monosdf - https://niujinshuchong.github.io/monosdf/
Enforces consistency between the volume rendered normal and the predicted monocular normal.
With both angluar and L1 loss. Eq 14 https://arxiv.org/pdf/2206.00665.pdf
Args:
normal_pred: volume rendered normal
normal_gt: monocular normal
"""
normal_gt = torch.nn.functional.normalize(normal_gt, p=2, dim=-1)
normal_pred = torch.nn.functional.normalize(normal_pred, p=2, dim=-1)
l1 = torch.abs(normal_pred - normal_gt).sum(dim=-1).mean()
cos = (1.0 - torch.sum(normal_pred * normal_gt, dim=-1)).mean()
return l1 + cos
class MiDaSMSELoss(nn.Module):
"""
data term from MiDaS paper
"""
def __init__(self, reduction_type: Literal["image", "batch"] = "batch"):
super().__init__()
self.reduction_type = reduction_type
# reduction here is different from the image/batch-based reduction. This is either "mean" or "sum"
self.mse_loss = MSELoss(reduction="none")
def forward(
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
) -> TensorType[0]:
"""
Args:
prediction: predicted depth map
target: ground truth depth map
mask: mask of valid pixels
Returns:
mse loss based on reduction function
"""
summed_mask = torch.sum(mask, (1, 2))
image_loss = torch.sum(self.mse_loss(prediction, target) * mask, (1, 2))
image_loss = masked_reduction(image_loss, summed_mask, self.reduction_type)
return image_loss
# losses based on https://github.com/autonomousvision/monosdf/blob/main/code/model/loss.py
class GradientLoss(nn.Module):
"""
multiscale, scale-invariant gradient matching term to the disparity space.
This term biases discontinuities to be sharp and to coincide with discontinuities in the ground truth
More info here https://arxiv.org/pdf/1907.01341.pdf Equation 11
"""
def __init__(self, scales: int = 4, reduction_type: Literal["image", "batch"] = "batch"):
"""
Args:
scales: number of scales to use
reduction_type: either "batch" or "image"
"""
super().__init__()
self.reduction_type = reduction_type
self.__scales = scales
def forward(
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
) -> TensorType[0]:
"""
Args:
prediction: predicted depth map
target: ground truth depth map
mask: mask of valid pixels
Returns:
gradient loss based on reduction function
"""
total = 0
for scale in range(self.__scales):
step = pow(2, scale)
grad_loss = self.gradient_loss(
prediction[:, ::step, ::step],
target[:, ::step, ::step],
mask[:, ::step, ::step],
)
total += grad_loss
return total
def gradient_loss(
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
) -> TensorType[0]:
"""
multiscale, scale-invariant gradient matching term to the disparity space.
This term biases discontinuities to be sharp and to coincide with discontinuities in the ground truth
More info here https://arxiv.org/pdf/1907.01341.pdf Equation 11
Args:
prediction: predicted depth map
target: ground truth depth map
reduction: reduction function, either reduction_batch_based or reduction_image_based
Returns:
gradient loss based on reduction function
"""
summed_mask = torch.sum(mask, (1, 2))
diff = prediction - target
diff = torch.mul(mask, diff)
grad_x = torch.abs(diff[:, :, 1:] - diff[:, :, :-1])
mask_x = torch.mul(mask[:, :, 1:], mask[:, :, :-1])
grad_x = torch.mul(mask_x, grad_x)
grad_y = torch.abs(diff[:, 1:, :] - diff[:, :-1, :])
mask_y = torch.mul(mask[:, 1:, :], mask[:, :-1, :])
grad_y = torch.mul(mask_y, grad_y)
image_loss = torch.sum(grad_x, (1, 2)) + torch.sum(grad_y, (1, 2))
image_loss = masked_reduction(image_loss, summed_mask, self.reduction_type)
return image_loss
class ScaleAndShiftInvariantLoss(nn.Module):
"""
Scale and shift invariant loss as described in
"Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer"
https://arxiv.org/pdf/1907.01341.pdf
"""
def __init__(self, alpha: float = 0.5, scales: int = 4, reduction_type: Literal["image", "batch"] = "batch"):
"""
Args:
alpha: weight of the regularization term
scales: number of scales to use
reduction_type: either "batch" or "image"
"""
super().__init__()
self.__data_loss = MiDaSMSELoss(reduction_type=reduction_type)
self.__regularization_loss = GradientLoss(scales=scales, reduction_type=reduction_type)
self.__alpha = alpha
self.__prediction_ssi = None
@classmethod
def compute_scale_and_shift(
cls, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
):
"""
More info here: https://arxiv.org/pdf/2206.00665.pdf supplementary section A2 Depth Consistency Loss
This function computes scale/shift required to normalizes predicted depth map,
to allow for using normalized depth maps as input from monocular depth estimation networks.
These networks are trained such that they predict normalized depth maps.
Solves for scale/shift using a least squares approach with a closed form solution:
Based on:
https://github.com/autonomousvision/monosdf/blob/d9619e948bf3d85c6adec1a643f679e2e8e84d4b/code/model/loss.py#L7
Args:
prediction: predicted depth map
target: ground truth depth map
mask: mask of valid pixels
Returns:
scale and shift for depth prediction
"""
# system matrix: A = [[a_00, a_01], [a_10, a_11]]
a_00 = torch.sum(mask * prediction * prediction, (1, 2))
a_01 = torch.sum(mask * prediction, (1, 2))
a_11 = torch.sum(mask, (1, 2))
# right hand side: b = [b_0, b_1]
b_0 = torch.sum(mask * prediction * target, (1, 2))
b_1 = torch.sum(mask * target, (1, 2))
# solution: x = A^-1 . b = [[a_11, -a_01], [-a_10, a_00]] / (a_00 * a_11 - a_01 * a_10) . b
scale = torch.zeros_like(b_0)
shift = torch.zeros_like(b_1)
det = a_00 * a_11 - a_01 * a_01
valid = det.nonzero()
scale[valid] = (a_11[valid] * b_0[valid] - a_01[valid] * b_1[valid]) / det[valid]
shift[valid] = (-a_01[valid] * b_0[valid] + a_00[valid] * b_1[valid]) / det[valid]
return scale, shift
def forward(
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
) -> TensorType[0]:
"""
Args:
prediction: predicted depth map (unnormalized)
target: ground truth depth map (normalized)
mask: mask of valid pixels
Returns:
scale and shift invariant loss
"""
scale, shift = ScaleAndShiftInvariantLoss.compute_scale_and_shift(prediction, target, mask)
self.__prediction_ssi = scale.view(-1, 1, 1) * prediction + shift.view(-1, 1, 1)
total = self.__data_loss(self.__prediction_ssi, target, mask)
if self.__alpha > 0:
total += self.__alpha * self.__regularization_loss(self.__prediction_ssi, target, mask)
return total
def __get_prediction_ssi(self):
"""
scale and shift invariant prediction
from https://arxiv.org/pdf/1907.01341.pdf equation 1
"""
return self.__prediction_ssi
prediction_ssi = property(__get_prediction_ssi)
+4 -3
View File
@@ -37,6 +37,7 @@ from torchtyping import TensorType
from typing_extensions import Literal
from nerfstudio.cameras.rays import RaySamples
from nerfstudio.utils import colors
from nerfstudio.utils.math import components_from_spherical_harmonics, safe_normalize
BACKGROUND_COLOR_OVERRIDE: Optional[TensorType[3]] = None
@@ -70,7 +71,7 @@ class RGBRenderer(nn.Module):
cls,
rgb: TensorType["bs":..., "num_samples", 3],
weights: TensorType["bs":..., "num_samples", 1],
background_color: Union[Literal["random", "black", "last_sample"], TensorType[3]] = "random",
background_color: Union[Literal["random", "white", "black", "last_sample"], TensorType[3]] = "random",
ray_indices: Optional[TensorType["num_samples"]] = None,
num_rays: Optional[int] = None,
) -> TensorType["bs":..., 3]:
@@ -102,8 +103,8 @@ class RGBRenderer(nn.Module):
background_color = rgb[..., -1, :]
if background_color == "random":
background_color = torch.rand_like(comp_rgb).to(rgb.device)
if background_color == "black":
background_color = torch.zeros_like(comp_rgb).to(rgb.device)
if isinstance(background_color, str) and background_color in colors.COLORS_DICT:
background_color = colors.COLORS_DICT[background_color].to(rgb.device)
assert isinstance(background_color, torch.Tensor)
comp_rgb = comp_rgb + background_color.to(weights.device) * (1.0 - accumulated_weight)
+2 -1
View File
@@ -70,7 +70,8 @@ class DepthNerfactoModel(NerfactoModel):
def get_outputs(self, ray_bundle: RayBundle):
outputs = super().get_outputs(ray_bundle)
outputs["directions_norm"] = ray_bundle.metadata["directions_norm"]
if ray_bundle.metadata is not None and "directions_norm" in ray_bundle.metadata:
outputs["directions_norm"] = ray_bundle.metadata["directions_norm"]
return outputs
def get_metrics_dict(self, outputs, batch):
+50 -44
View File
@@ -19,12 +19,12 @@ DreamFusion implementation.
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Dict, List, Tuple, Type, Optional
from typing import Dict, List, Optional, Tuple, Type
import numpy as np
import torch
from torch.nn import Parameter
from torch.cuda.amp.grad_scaler import GradScaler
from typing_extensions import Literal
from nerfstudio.cameras.rays import RayBundle
from nerfstudio.engine.callbacks import (
@@ -32,11 +32,11 @@ from nerfstudio.engine.callbacks import (
TrainingCallbackAttributes,
TrainingCallbackLocation,
)
from nerfstudio.generative.stable_diffusion import StableDiffusion
from nerfstudio.generative.stable_diffusion_utils import get_text_embedding
from nerfstudio.field_components.field_heads import FieldHeadNames
from nerfstudio.fields.density_fields import HashMLPDensityField
from nerfstudio.fields.dreamfusion_field import DreamFusionField
from nerfstudio.generative.stable_diffusion import StableDiffusion
from nerfstudio.generative.stable_diffusion_utils import PositionalTextEmbeddings
from nerfstudio.model_components.losses import (
MSELoss,
distortion_loss,
@@ -77,7 +77,7 @@ class DreamFusionModelConfig(ModelConfig):
"""Randomizes light source per output."""
initialize_density: bool = True
"""Initialize density in center of scene."""
init_density_strength: float = 0.1
init_density_strength: float = 10
"""Initial strength of center density"""
sphere_collider: bool = True
"""Use spherical collider instead of box"""
@@ -135,8 +135,8 @@ class DreamFusionModelConfig(ModelConfig):
"""enables location based prompting"""
interpolated_prompting: bool = False
"""enables interpolated location prompting"""
prompting_type: str = "location_based"
"""choose between location_based, interpolated, base"""
positional_prompting: Literal["discrete", "interpolated", "base"] = "discrete"
""" how to incorporate position into prompt"""
top_prompt: str = ", overhead view"
"""appended to prompt for overhead view"""
side_prompt: str = ", side view"
@@ -149,7 +149,8 @@ class DreamFusionModelConfig(ModelConfig):
"""guidance scale for sds loss"""
stablediffusion_device: Optional[str] = None
"""device for stable diffusion"""
sd_version: str = '1-5'
sd_version: str = "1-5"
class DreamFusionModel(Model):
"""DreamFusionModel Model
@@ -174,17 +175,17 @@ class DreamFusionModel(Model):
self.random_background = config.random_background
self.density_strength = 1.0
self.target_transmittance = config.target_transmittance_start
self.grad_scaler = kwargs['grad_scaler']
self.grad_scaler = kwargs["grad_scaler"]
self.prompting_type = config.prompting_type
self.positional_prompting = config.positional_prompting
self.guidance_scale = config.guidance_scale
self.top_prompt = config.top_prompt
self.side_prompt = config.side_prompt
self.back_prompt = config.back_prompt
self.back_prompt = config.back_prompt
self.front_prompt = config.front_prompt
self.sd_device = (
torch.device(kwargs['device'])
torch.device(kwargs["device"])
if config.stablediffusion_device is None
else torch.device(config.stablediffusion_device)
)
@@ -196,13 +197,15 @@ class DreamFusionModel(Model):
super().populate_modules()
self.sd = StableDiffusion(self.sd_device, version=self.sd_version)
self.text_embeddings = {
"top_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.top_prompt}", ""),
"front_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.front_prompt}", ""),
"side_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.side_prompt}", ""),
"back_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.back_prompt}", ""),
"base_text_embedding": self.sd.get_text_embeds(self.cur_prompt, "")
}
self.text_embeddings = PositionalTextEmbeddings(
base_prompt=self.cur_prompt,
top_prompt=self.cur_prompt + self.top_prompt,
side_prompt=self.cur_prompt + self.side_prompt,
back_prompt=self.cur_prompt + self.back_prompt,
front_prompt=self.cur_prompt + self.front_prompt,
stable_diffusion=self.sd,
positional_prompting=self.positional_prompting,
)
# setting up fields
self.field = DreamFusionField(self.scene_box.aabb, max_res=self.config.max_res)
@@ -339,25 +342,26 @@ class DreamFusionModel(Model):
param_groups["fields"] = list(self.field.parameters())
return param_groups
def get_outputs(self, ray_bundle: RayBundle):
def get_outputs(self, ray_bundle: RayBundle): # pylint: disable=too-many-statements
# uniform sampling
background_rgb = self.field.get_background_rgb(ray_bundle)
ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns)
field_outputs = self.field(ray_samples, compute_normals=True)
weights = ray_samples.get_weights(field_outputs[FieldHeadNames.DENSITY])
weights_list.append(weights)
ray_samples_list.append(ray_samples)
density = field_outputs[FieldHeadNames.DENSITY]
if self.initialize_density and self.training:
if self.initialize_density:
pos = ray_samples.frustums.get_positions()
density_blob = (
self.config.init_density_strength
* self.density_strength
* torch.exp(-torch.norm(pos, dim=-1) / (2 * 0.04))[..., None]
* torch.exp(-torch.norm(pos, dim=-1) * 10)[..., None]
)
density += density_blob
weights = ray_samples.get_weights(density)
weights_list.append(weights)
ray_samples_list.append(ray_samples)
accumulation = self.renderer_accumulation(weights)
depth = self.renderer_depth(weights, ray_samples)
rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights)
@@ -421,7 +425,7 @@ class DreamFusionModel(Model):
else:
outputs["train_output"] = outputs["rgb"]
if self.training or True:
if self.training:
outputs["rendered_orientation_loss"] = orientation_loss(
weights.detach(), field_outputs[FieldHeadNames.NORMALS], ray_bundle.directions
)
@@ -463,28 +467,30 @@ class DreamFusionModel(Model):
if self.prompt != self.cur_prompt:
self.cur_prompt = self.prompt
self.text_embeddings = {
"top_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.top_prompt}", ""),
"front_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.front_prompt}", ""),
"side_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.side_prompt}", ""),
"back_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.back_prompt}", ""),
"base_text_embedding": self.sd.get_text_embeds(self.cur_prompt, "")
}
text_embedding = get_text_embedding(batch, self.prompting_type, self.text_embeddings, self.sd_device)
self.text_embeddings.update_prompt(
base_prompt=self.cur_prompt,
top_prompt=self.cur_prompt + self.top_prompt,
side_prompt=self.cur_prompt + self.side_prompt,
back_prompt=self.cur_prompt + self.back_prompt,
front_prompt=self.cur_prompt + self.front_prompt,
)
text_embedding = self.text_embeddings.get_text_embedding(
vertical_angle=batch["vertical"], horizonal_angle=batch["central"]
)
train_output = (
outputs["train_output"].view(1, int(outputs["train_output"].shape[0] ** 0.5), int(outputs["train_output"].shape[0] ** 0.5), 3)
outputs["train_output"]
.view(1, int(outputs["train_output"].shape[0] ** 0.5), int(outputs["train_output"].shape[0] ** 0.5), 3)
.permute(0, 3, 1, 2)
)
with torch.autocast(device_type="cuda", dtype=torch.float16):
sds_loss = self.sd.sds_loss(
text_embedding.to(self.sd_device),
train_output.to(self.sd_device),
guidance_scale=int(self.guidance_scale),
grad_scaler=self.grad_scaler,
)
sds_loss = self.sd.sds_loss(
text_embedding.to(self.sd_device),
train_output.to(self.sd_device),
guidance_scale=int(self.guidance_scale),
grad_scaler=self.grad_scaler,
)
loss_dict["sds_loss"] = sds_loss.to(self.device)
+2 -6
View File
@@ -46,7 +46,7 @@ from nerfstudio.model_components.renderers import (
RGBRenderer,
)
from nerfstudio.models.base_model import Model, ModelConfig
from nerfstudio.utils import colormaps, colors
from nerfstudio.utils import colormaps
@dataclass
@@ -123,11 +123,7 @@ class NGPModel(Model):
)
# renderers
background_color = "random"
if self.config.background_color in ["white", "black"]:
background_color = colors.COLORS_DICT[self.config.background_color]
self.renderer_rgb = RGBRenderer(background_color=background_color)
self.renderer_rgb = RGBRenderer(background_color=self.config.background_color)
self.renderer_accumulation = AccumulationRenderer()
self.renderer_depth = DepthRenderer(method="expected")
+1 -1
View File
@@ -68,7 +68,7 @@ class NerfactoModelConfig(ModelConfig):
"""How far along the ray to start sampling."""
far_plane: float = 1000.0
"""How far along the ray to stop sampling."""
background_color: Literal["random", "last_sample"] = "last_sample"
background_color: Literal["random", "last_sample", "black", "white"] = "last_sample"
"""Whether to randomize the background color."""
num_levels: int = 16
"""Number of levels of the hashmap for the base mlp."""
+11 -3
View File
@@ -50,6 +50,7 @@ from nerfstudio.model_components.renderers import (
RGBRenderer,
)
from nerfstudio.model_components.scene_colliders import NearFarCollider
from nerfstudio.model_components.shaders import NormalsShader
from nerfstudio.models.base_model import Model
from nerfstudio.models.nerfacto import NerfactoModel, NerfactoModelConfig
@@ -63,7 +64,7 @@ class NerfplayerNerfactoModelConfig(NerfactoModelConfig):
"""How far along the ray to start sampling."""
far_plane: float = 1000.0
"""How far along the ray to stop sampling."""
background_color: Literal["random", "last_sample"] = "random"
background_color: Literal["random", "last_sample", "black", "white"] = "random"
"""Whether to randomize the background color. (Random is reported to be better on DyCheck.)"""
num_levels: int = 16
"""Hashing grid parameter."""
@@ -162,6 +163,9 @@ class NerfplayerNerfactoModel(NerfactoModel):
self.renderer_depth = DepthRenderer(method="expected") # for depth loss
self.renderer_normals = NormalsRenderer()
# shaders
self.normals_shader = NormalsShader()
# losses
self.rgb_loss = MSELoss()
@@ -192,8 +196,12 @@ class NerfplayerNerfactoModel(NerfactoModel):
}
if self.config.predict_normals:
outputs["normals"] = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights)
outputs["pred_normals"] = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights)
outputs["normals"] = self.normals_shader(
self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights)
)
outputs["pred_normals"] = self.normals_shader(
self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights)
)
# These use a lot of GPU memory, so we avoid storing them for eval.
if self.training:
+2 -10
View File
@@ -42,7 +42,6 @@ from nerfstudio.model_components.renderers import (
)
from nerfstudio.models.base_model import Model
from nerfstudio.models.instant_ngp import InstantNGPModelConfig, NGPModel
from nerfstudio.utils import colors
@dataclass
@@ -128,13 +127,6 @@ class NerfplayerNGPModel(NGPModel):
) # need to update the density_fn later during forward (for input time)
# renderers
self.train_background_color = self.config.train_background_color
self.eval_background_color = self.config.eval_background_color
if self.config.train_background_color in ["white", "black"]:
self.train_background_color = colors.COLORS_DICT[self.config.train_background_color]
if self.config.eval_background_color in ["white", "black"]:
self.eval_background_color = colors.COLORS_DICT[self.config.eval_background_color]
self.renderer_rgb = RGBRenderer() # will update bgcolor later during forward
self.renderer_accumulation = AccumulationRenderer()
self.renderer_depth = DepthRenderer(method="expected")
@@ -175,9 +167,9 @@ class NerfplayerNGPModel(NGPModel):
# update bgcolor in the renderer; usually random color for training and fixed color for inference
if self.training:
self.renderer_rgb.background_color = self.train_background_color
self.renderer_rgb.background_color = self.config.train_background_color
else:
self.renderer_rgb.background_color = self.eval_background_color
self.renderer_rgb.background_color = self.config.eval_background_color
rgb = self.renderer_rgb(
rgb=field_outputs[FieldHeadNames.RGB],
weights=weights,
+1 -1
View File
@@ -237,7 +237,7 @@ class VanillaPipeline(Pipeline):
num_train_data=len(self.datamanager.train_dataset),
metadata=self.datamanager.train_dataset.metadata,
device=device,
grad_scaler=grad_scaler
grad_scaler=grad_scaler,
)
self.model.to(device)
+1 -10
View File
@@ -604,14 +604,6 @@ def colmap_to_json(
cameras = read_cameras_binary(cameras_path)
images = read_images_binary(images_path)
# Images were renamed to frame_{i:05d}.{ext} and
# the filenames needs to be replaced in the transforms.json as well
original_filenames = [x.name for x in images.values()]
# Sort was used in nerfstudio.process_data.process_data_utils:get_image_filenames
original_filenames.sort()
# Build the map to the new filenames
filename_map = {name: f"frame_{i+1:05d}{os.path.splitext(name)[-1]}" for i, name in enumerate(original_filenames)}
# Only supports one camera
camera_params = cameras[1].params
@@ -627,8 +619,7 @@ def colmap_to_json(
c2w = c2w[np.array([1, 0, 2, 3]), :]
c2w[2, :] *= -1
name = filename_map[im_data.name]
name = Path(f"./images/{name}")
name = Path(f"./images/{im_data.name}")
frame = {
"file_path": name.as_posix(),
+26
View File
@@ -19,6 +19,7 @@ from typing import Tuple
import torch
from torchtyping import TensorType
from typing_extensions import Literal
_USE_NERFACC = True
@@ -285,3 +286,28 @@ def safe_normalize(
Normalized vectors.
"""
return vectors / (torch.norm(vectors, dim=-1, keepdim=True) + eps)
def masked_reduction(
input_tensor: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"], reduction_type: Literal["image", "batch"]
):
"""
Whether to consolidate the input_tensor across the batch or across the image
Args:
input_tensor: input tensor
mask: mask tensor
reduction_type: either "batch" or "image"
Returns:
input_tensor: reduced input_tensor
"""
if reduction_type == "batch":
# avoid division by 0 (if sum(M) = sum(sum(mask)) = 0: sum(image_loss) = 0)
divisor = torch.sum(mask)
input_tensor = 0 if divisor == 0 else torch.sum(input_tensor) / divisor
elif reduction_type == "image":
# avoid division by 0 (if M = sum(mask) = 0: image_loss = 0)
valid = mask.nonzero()
input_tensor[valid] = input_tensor[valid] / mask[valid]
input_tensor = torch.mean(input_tensor)
return input_tensor
+3 -1
View File
@@ -12,6 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# pylint: disable=too-many-lines
"""Code to interface with the `vis/` (the JS viewer).
"""
from __future__ import annotations
@@ -494,7 +496,7 @@ class ViewerState:
prompt = getattr(graph, "prompt", None)
is_generative = prompt is not None
if is_generative:
self.vis["model/is_generative"].write(str(is_generative).lower())
+2 -2
View File
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "nerfstudio"
version = "0.1.17"
version = "0.1.18"
description = "All-in-one repository for state-of-the-art NeRFs"
readme = "README.md"
license = { text="Apache 2.0"}
@@ -100,7 +100,7 @@ ns-install-cli = "scripts.completions.install:entrypoint"
ns-process-data = "scripts.process_data:entrypoint"
ns-download-data = "scripts.downloads.download_data:entrypoint"
ns-train = "scripts.train:entrypoint"
ns-trace-sd = "scripts.trace_stablediff:entrypoint"
ns-trace-sd = "scripts.generative.trace_stable_diffusion:entrypoint"
ns-viewer = "scripts.viewer.run_viewer:entrypoint"
ns-eval = "scripts.eval:entrypoint"
ns-render = "scripts.render:entrypoint"
View File
File mode changed.
+1 -1
View File
@@ -28,7 +28,7 @@ fi
method_opts=()
if [ "$method_name" = "nerfacto" ]; then
# https://github.com/nerfstudio-project/nerfstudio/issues/806#issuecomment-1284327844
method_opts=(--pipeline.model.near-plane 2. --pipeline.model.far-plane 6. --pipeline.datamanager.camera-optimizer.mode off --pipeline.model.use-average-appearance-embedding False)
method_opts=(--pipeline.model.background-color white --pipeline.model.near-plane 2. --pipeline.model.far-plane 6. --pipeline.datamanager.camera-optimizer.mode off --pipeline.model.use-average-appearance-embedding False)
fi
shift $((OPTIND-1))
Executable → Regular
+199 -84
View File
@@ -10,27 +10,29 @@ import shutil
import stat
import subprocess
import sys
from typing import List, Union
from typing import List, Optional, Union
import tyro
from rich.console import Console
from rich.prompt import Confirm
from typing_extensions import Literal, assert_never
from typing_extensions import get_args as typing_get_args
if sys.version_info < (3, 10):
import importlib_metadata
else:
from importlib import metadata as importlib_metadata
ConfigureMode = Literal["install", "uninstall"]
ShellType = Literal["zsh", "bash"]
CONSOLE = Console(width=120)
HEADER_LINE = "# Source nerfstudio autocompletions."
ENTRYPOINTS = [
"ns-install-cli",
"ns-process-data",
"ns-download-data",
"ns-train",
"ns-eval",
"ns-render",
"ns-dev-test",
]
def _get_all_entry_points() -> List[str]:
entry_points = importlib_metadata.distribution("nerfstudio").entry_points
return [x.name for x in entry_points]
def _check_tyro_cli(script_path: pathlib.Path) -> bool:
@@ -127,6 +129,62 @@ def _exclamation() -> str:
return random.choice(["Cool", "Nice", "Neat", "Great", "Exciting", "Excellent", "Ok"]) + "!"
def _get_deactivate_script(commands: List[str], shell: Optional[ShellType], add_header=True) -> str:
if shell is None:
# Install the universal script
result_script = []
for shell_type in typing_get_args(ShellType):
result_script.append(f'if [ -n "${shell_type.upper()}_VERSION" ]; then')
result_script.append(_get_deactivate_script(commands, shell_type, add_header=False))
result_script.append("fi")
source_lines = "\n".join(result_script)
elif shell == "zsh":
source_lines = "\n".join([f"unset '_comps[{command}]' &> /dev/null" for command in commands])
elif shell == "bash":
source_lines = "\n".join([f"complete -r {command} &> /dev/null" for command in commands])
else:
assert_never(shell)
if add_header:
source_lines = f"\n{HEADER_LINE}\n{source_lines}"
return source_lines
def _get_source_script(completions_dir: pathlib.Path, shell: Optional[ShellType], add_header=True) -> str:
if shell is None:
# Install the universal script
result_script = []
for shell_type in typing_get_args(ShellType):
result_script.append(f'if [ -n "${shell_type.upper()}_VERSION" ]; then')
result_script.append(_get_source_script(completions_dir, shell_type, add_header=False))
result_script.append("fi")
source_lines = "\n".join(result_script)
elif shell == "zsh":
source_lines = "\n".join(
[
"if ! command -v compdef &> /dev/null; then",
" autoload -Uz compinit",
" compinit",
"fi",
f"source {completions_dir / 'setup.zsh'}",
]
)
elif shell == "bash":
source_lines = "\n".join(
[
f"source {completions_dir / 'setup.bash'}",
]
)
else:
assert_never(shell)
if add_header:
source_lines = f"\n{HEADER_LINE}\n{source_lines}"
return source_lines
def _update_rc(
completions_dir: pathlib.Path,
mode: ConfigureMode,
@@ -141,36 +199,13 @@ def _update_rc(
"""
# Install or uninstall `source_line`.
header_line = "# Source nerfstudio autocompletions."
if shell == "zsh":
source_lines = "\n".join(
[
"",
header_line,
"if ! command -v compdef &> /dev/null; then",
" autoload -Uz compinit",
" compinit",
"fi",
f"source {completions_dir / 'setup.zsh'}",
]
)
elif shell == "bash":
source_lines = "\n".join(
[
"",
header_line,
f"source {completions_dir / 'setup.bash'}",
]
)
else:
assert_never(shell)
source_lines = _get_source_script(completions_dir, shell)
rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
# Always try to uninstall previous completions.
rc_source = rc_path.read_text()
while header_line in rc_source:
before_install, _, after_install = rc_source.partition(header_line)
while HEADER_LINE in rc_source:
before_install, _, after_install = rc_source.partition(HEADER_LINE)
source_file, _, after_install = after_install.partition("\nsource ")[2].partition("\n")
assert source_file.endswith(f"/completions/setup.{shell}")
rc_source = before_install + after_install
@@ -189,6 +224,108 @@ def _update_rc(
assert mode == "uninstall"
def _update_conda_scripts(
commands: List[str],
completions_dir: pathlib.Path,
mode: ConfigureMode,
) -> None:
"""Try to add a `source /.../completions/setup.{shell}` line automatically to conda's activation scripts.
Args:
completions_dir: Path to location of this script.
mode: Install or uninstall completions.
"""
# Install or uninstall `source_line`.
activate_source_lines = _get_source_script(completions_dir, None)
deactivate_source_lines = _get_deactivate_script(commands, None)
conda_path = pathlib.Path(os.environ["CONDA_PREFIX"])
activate_path = conda_path / "etc/conda/activate.d/nerfstudio_activate.sh"
deactivate_path = conda_path / "etc/conda/deactivate.d/nerfstudio_deactivate.sh"
if mode == "uninstall":
if activate_path.exists():
os.remove(activate_path)
if deactivate_path.exists():
os.remove(deactivate_path)
CONSOLE.log(f":broom: Existing completions uninstalled from {conda_path}.")
elif mode == "install":
# Install completions.
activate_path.parent.mkdir(exist_ok=True, parents=True)
deactivate_path.parent.mkdir(exist_ok=True, parents=True)
with activate_path.open("w+", encoding="utf8") as f:
f.write(activate_source_lines)
with deactivate_path.open("w+", encoding="utf8") as f:
f.write(deactivate_source_lines)
CONSOLE.log(
f":person_gesturing_ok: Completions installed to {conda_path}. {_exclamation()} Reactivate the environment"
" to try them out."
)
else:
assert_never(mode)
def _get_conda_path() -> Optional[pathlib.Path]:
"""
Returns the path to the conda environment if
the nerfstudio package is installed in one.
"""
conda_path = None
if "CONDA_PREFIX" in os.environ:
# Conda is active, we will check if the Nerfstudio is installed in the conda env.
distribution = importlib_metadata.distribution("nerfstudio")
if str(distribution.locate_file("nerfstudio")).startswith(os.environ["CONDA_PREFIX"]):
conda_path = pathlib.Path(os.environ["CONDA_PREFIX"])
return conda_path
def _generate_completions_files(
completions_dir: pathlib.Path,
scripts_dir: pathlib.Path,
shells_supported: List[ShellType],
shells_found: List[ShellType],
) -> None:
# Set to True to install completions for scripts as well.
include_scripts = False
# Find tyro CLIs.
script_paths = list(filter(_check_tyro_cli, scripts_dir.glob("**/*.py"))) if include_scripts else []
script_names = tuple(p.name for p in script_paths)
assert len(set(script_names)) == len(script_names)
# Get existing completion files.
existing_completions = set()
for shell in shells_supported:
target_dir = completions_dir / shell
if target_dir.exists():
existing_completions |= set(target_dir.glob("*"))
# Get all entry_points.
entry_points = _get_all_entry_points()
# Run generation jobs.
concurrent_executor = concurrent.futures.ThreadPoolExecutor()
with CONSOLE.status("[bold]:writing_hand: Generating completions...", spinner="bouncingBall"):
completion_paths = list(
concurrent_executor.map(
lambda path_or_entrypoint_and_shell: _generate_completion(
path_or_entrypoint_and_shell[0], path_or_entrypoint_and_shell[1], completions_dir
),
itertools.product(script_paths + entry_points, shells_found),
)
)
# Delete obsolete completion files.
for unexpected_path in set(p.absolute() for p in existing_completions) - set(
p.absolute() for p in completion_paths
):
if unexpected_path.is_dir():
shutil.rmtree(unexpected_path)
elif unexpected_path.exists():
unexpected_path.unlink()
CONSOLE.log(f":broom: Deleted {unexpected_path}.")
def main(mode: ConfigureMode = "install") -> None:
"""Main script.
@@ -201,16 +338,24 @@ def main(mode: ConfigureMode = "install") -> None:
CONSOLE.log("[bold red]$HOME is not set. Exiting.")
return
# Get conda path if in conda environment.
conda_path = _get_conda_path()
# Try to locate the user's bashrc or zshrc.
shells_supported: List[ShellType] = ["zsh", "bash"]
shells_found: List[ShellType] = []
for shell in shells_supported:
rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
if not rc_path.exists():
CONSOLE.log(f":person_shrugging: {rc_path.name} not found, skipping.")
else:
CONSOLE.log(f":mag: Found {rc_path.name}!")
shells_found.append(shell)
shells_supported: List[ShellType] = list(typing_get_args(ShellType))
if conda_path is not None:
# Running in conda; we have to support all shells.
shells_found = shells_supported
CONSOLE.log(f":mag: Detected conda environment {conda_path}!")
else:
shells_found: List[ShellType] = []
for shell in shells_supported:
rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
if not rc_path.exists():
CONSOLE.log(f":person_shrugging: {rc_path.name} not found, skipping.")
else:
CONSOLE.log(f":mag: Found {rc_path.name}!")
shells_found.append(shell)
# Get scripts/ directory.
completions_dir = pathlib.Path(__file__).absolute().parent
@@ -230,48 +375,18 @@ def main(mode: ConfigureMode = "install") -> None:
else:
CONSOLE.log(f":heavy_check_mark: No existing completions at: {target_dir}.")
elif mode == "install":
# Set to True to install completions for scripts as well.
include_scripts = False
# Find tyro CLIs.
script_paths = list(filter(_check_tyro_cli, scripts_dir.glob("**/*.py"))) if include_scripts else []
script_names = tuple(p.name for p in script_paths)
assert len(set(script_names)) == len(script_names)
# Get existing completion files.
existing_completions = set()
for shell in shells_supported:
target_dir = completions_dir / shell
if target_dir.exists():
existing_completions |= set(target_dir.glob("*"))
# Run generation jobs.
concurrent_executor = concurrent.futures.ThreadPoolExecutor()
with CONSOLE.status("[bold]:writing_hand: Generating completions...", spinner="bouncingBall"):
completion_paths = list(
concurrent_executor.map(
lambda path_or_entrypoint_and_shell: _generate_completion(
path_or_entrypoint_and_shell[0], path_or_entrypoint_and_shell[1], completions_dir
),
itertools.product(script_paths + ENTRYPOINTS, shells_found),
)
)
# Delete obsolete completion files.
for unexpected_path in set(p.absolute() for p in existing_completions) - set(
p.absolute() for p in completion_paths
):
if unexpected_path.is_dir():
shutil.rmtree(unexpected_path)
elif unexpected_path.exists():
unexpected_path.unlink()
CONSOLE.log(f":broom: Deleted {unexpected_path}.")
_generate_completions_files(completions_dir, scripts_dir, shells_supported, shells_found)
else:
assert_never(mode)
# Install or uninstall from bashrc/zshrc.
for shell in shells_found:
_update_rc(completions_dir, mode, shell)
if conda_path is not None:
# In conda environment we add the completitions activation scripts.
commands = _get_all_entry_points()
_update_conda_scripts(commands, completions_dir, mode)
else:
# Install or uninstall from bashrc/zshrc.
for shell in shells_found:
_update_rc(completions_dir, mode, shell)
CONSOLE.print("[bold]All done![/bold]")
Executable → Regular
View File
File mode changed.
Executable → Regular
View File
File mode changed.
View File
File mode changed.
Executable → Regular
View File
File mode changed.
+65 -40
View File
@@ -2,6 +2,7 @@
import functools
import sys
from dataclasses import dataclass
from pathlib import Path
import appdirs
@@ -9,6 +10,8 @@ import torch
import tyro
from rich.console import Console
from nerfstudio.generative.stable_diffusion import SD_IDENTIFIERS
CONSOLE = Console(width=120)
try:
@@ -20,55 +23,77 @@ except ImportError:
sys.exit(1)
def jit_unet(save_dir: Path = Path(appdirs.user_data_dir("nerfstudio"))):
"""Trace Stable Diffusion UNet for speed improvement
@dataclass
class TraceSD:
"""Traces the Stable Diffusion U-Net for better performance."""
Args:
save_dir: directory to save the traced model
"""
sd_version: str
"""Stable diffusion version."""
save_dir: Path = Path(appdirs.user_data_dir("nerfstudio"))
"""Directory to save the traced model."""
# torch disable grad
torch.set_grad_enabled(False)
def main(self):
"""Trace the Stable Diffusion U-Net and save it to disk."""
# torch disable grad
torch.set_grad_enabled(False)
# load inputs
def generate_inputs():
sample = torch.randn(2, 4, 64, 64).half().cuda()
timestep = torch.rand(1).half().cuda() * 999
encoder_hidden_states = torch.randn(2, 77, 768).half().cuda()
return sample, timestep, encoder_hidden_states
if self.sd_version not in SD_IDENTIFIERS:
CONSOLE.print(f"[bold red]Invalid Stable Diffusion version; choose from {set(SD_IDENTIFIERS.keys())}")
sys.exit(1)
pipe = StableDiffusionPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5",
torch_dtype=torch.float16,
).to("cuda")
sd_id = SD_IDENTIFIERS[self.sd_version]
pipe = StableDiffusionPipeline.from_pretrained(
sd_id,
torch_dtype=torch.float16,
)
assert pipe is not None
pipe = pipe.to("cuda")
pipe.enable_attention_slicing()
pipe.enable_attention_slicing()
# load inputs
def generate_inputs():
sample = torch.randn(2, 4, 64, 64).half().cuda()
timestep = torch.rand(1).half().cuda() * 999
unet = pipe.unet
unet.eval()
unet.to(memory_format=torch.channels_last) # use channels_last memory format
unet.forward = functools.partial(unet.forward, return_dict=False) # set return_dict=False as default
# get shape of encoder hidden state (varies by SD version)
text_input = pipe.tokenizer("", return_tensors="pt")
_, w, d = pipe.text_encoder(text_input.input_ids.cuda())[0].shape
# warmup
inputs = None
for _ in range(3):
with torch.inference_mode():
inputs = generate_inputs()
_ = unet(*inputs)
encoder_hidden_states = torch.randn(2, w, d).half().cuda()
return sample, timestep, encoder_hidden_states
# trace
print("tracing..")
assert inputs is not None
unet_traced = torch.jit.trace(unet, inputs)
unet_traced.eval()
print("done tracing")
unet = pipe.unet
unet.eval()
unet.to(memory_format=torch.channels_last) # use channels_last memory format
unet.forward = functools.partial(unet.forward, return_dict=False) # set return_dict=False as default
# save the traced model
unet_traced_filename = save_dir / "sd_unet_traced.pt"
if not unet_traced_filename.exists():
unet_traced_filename.parent.mkdir(parents=True, exist_ok=True)
unet_traced.save(unet_traced_filename)
# warmup
inputs = None
for _ in range(3):
with torch.inference_mode():
inputs = generate_inputs()
_ = unet(*inputs)
# trace
print("tracing..")
assert inputs is not None
unet_traced = torch.jit.trace(unet, inputs)
unet_traced.eval() # type: ignore
print("done tracing")
# save the traced model
filename_sd_id = sd_id.split("/")[-1]
unet_traced_filename = self.save_dir / f"{filename_sd_id}_unet_traced.pt"
if not unet_traced_filename.exists():
unet_traced_filename.parent.mkdir(parents=True, exist_ok=True)
unet_traced.save(unet_traced_filename) # type: ignore
def entrypoint():
"""Entrypoint for use with pyproject scripts."""
tyro.extras.set_accent_color("bright_yellow")
tyro.cli(TraceSD).main()
if __name__ == "__main__":
tyro.cli(jit_unet)
entrypoint()
Executable → Regular
View File
File mode changed.
Executable → Regular
View File
File mode changed.
-82
View File
@@ -1,82 +0,0 @@
"""Trace Stable Diffusion UNet for speed improvement"""
import functools
import sys
from dataclasses import dataclass
from pathlib import Path
import appdirs
import torch
import tyro
from diffusers import StableDiffusionPipeline
from rich.console import Console
from nerfstudio.generative.stable_diffusion import SD_IDENTIFIERS
CONSOLE = Console(width=120)
@dataclass
class TraceSD:
"""Traces the Stable Diffusion U-Net for better performance."""
sd_version: str
"""Stable diffusion version."""
def main(self):
# torch disable grad
torch.set_grad_enabled(False)
if self.sd_version not in SD_IDENTIFIERS.keys():
CONSOLE.print(f"[bold red]Invalid Stable Diffusion version; choose from {set(SD_IDENTIFIERS.keys())}")
sys.exit(1)
sd_id = SD_IDENTIFIERS[self.sd_version]
pipe = StableDiffusionPipeline.from_pretrained(
sd_id,
torch_dtype=torch.float16,
).to("cuda")
pipe.enable_attention_slicing()
# load inputs
def generate_inputs():
sample = torch.randn(2, 4, 64, 64).half().cuda()
timestep = torch.rand(1).half().cuda() * 999
# get shape of encoder hidden state (varies by SD version)
text_input = pipe.tokenizer("", return_tensors="pt")
_, w, d = pipe.text_encoder(text_input.input_ids.cuda())[0].shape
encoder_hidden_states = torch.randn(2, w, d).half().cuda()
return sample, timestep, encoder_hidden_states
unet = pipe.unet
unet.eval()
unet.to(memory_format=torch.channels_last) # use channels_last memory format
unet.forward = functools.partial(unet.forward, return_dict=False) # set return_dict=False as default
# warmup
for _ in range(3):
with torch.inference_mode():
inputs = generate_inputs()
_ = unet(*inputs)
# trace
print("tracing..")
unet_traced = torch.jit.trace(unet, inputs)
unet_traced.eval()
print("done tracing")
# save the traced model
filename_sd_id = sd_id.split('/')[-1]
unet_traced_filename = Path(appdirs.user_data_dir("nerfstudio")) / f'{filename_sd_id}_unet_traced.pt'
if not unet_traced_filename.exists():
unet_traced_filename.parent.mkdir(parents=True, exist_ok=True)
unet_traced.save(unet_traced_filename)
def entrypoint():
"""Entrypoint for use with pyproject scripts."""
tyro.extras.set_accent_color("bright_yellow")
tyro.cli(TraceSD).main()
if __name__ == "__main__":
entrypoint()
Executable → Regular
View File
File mode changed.
View File
File mode changed.