mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-16 21:03:46 +08:00
Merge branch 'generative' of https://github.com/plenoptix/nerfactory into terrance/single_img_generative
This commit is contained in:
commit
0e4dcb73c3
42 files changed
+877
-383
No files matched your search
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"name": "NerfStudio Dev Container",
|
||||
"dockerComposeFile": "docker-compose.yml",
|
||||
"service": "nerfstudio",
|
||||
"workspaceFolder": "/workspace",
|
||||
"shutdownAction": "stopCompose",
|
||||
"extensions": [
|
||||
"mutantdino.resourcemonitor",
|
||||
"ms-azuretools.vscode-docker",
|
||||
"nvidia.nsight-vscode-edition",
|
||||
"ms-python.python"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
version: "3.8"
|
||||
|
||||
services:
|
||||
nerfstudio:
|
||||
shm_size: '12gb'
|
||||
image: dromni/nerfstudio:0.1.17
|
||||
# build:
|
||||
# context: ..
|
||||
# dockerfile: ./Dockerfile
|
||||
stdin_open: true
|
||||
tty: true
|
||||
environment:
|
||||
NVIDIA_DRIVER_CAPABILITIES: compute,utility,graphics
|
||||
DISPLAY: $DISPLAY
|
||||
ports:
|
||||
- "7007:7007"
|
||||
volumes:
|
||||
- ../:/workspace/
|
||||
- /tmp/.X11-unix:/tmp/.X11-unix
|
||||
working_dir: /workspace/
|
||||
command: /bin/bash
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [ gpu ]
|
||||
+27
-13
@@ -1,14 +1,16 @@
|
||||
# Define base image.
|
||||
FROM nvidia/cuda:11.7.1-devel-ubuntu22.04
|
||||
FROM nvidia/cuda:11.8.0-devel-ubuntu22.04
|
||||
|
||||
# Variables used at build time.
|
||||
## CUDA architectures, required by Colmap and tiny-cuda-nn.
|
||||
## NOTE: All commonly used GPU architectures are included and supported here. To speedup the image build process remove all architectures but the one of your explicit GPU. Find details here: https://developer.nvidia.com/cuda-gpus (8.6 translates to 86 in the line below) or in the docs.
|
||||
ARG CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37
|
||||
|
||||
# Set environment variables.
|
||||
## Set non-interactive to prevent asking for user inputs blocking image creation.
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
## Set timezone as it is required by some packages.
|
||||
ENV TZ=Europe/Berlin
|
||||
## CUDA architectures, required by tiny-cuda-nn.
|
||||
## NOTE: All commonly used GPU architectures are included and supported here. To speedup the image build process remove all architectures but the one of your explicit GPU. Find details here: https://developer.nvidia.com/cuda-gpus (8.6 translates to 86 in the line below) or in the docs.
|
||||
ENV TCNN_CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37
|
||||
## CUDA Home, required to find CUDA in some packages.
|
||||
ENV CUDA_HOME="/usr/local/cuda"
|
||||
|
||||
@@ -17,6 +19,7 @@ RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
cmake \
|
||||
curl \
|
||||
ffmpeg \
|
||||
git \
|
||||
libatlas-base-dev \
|
||||
@@ -27,6 +30,7 @@ RUN apt-get update && \
|
||||
libboost-test-dev \
|
||||
libcgal-dev \
|
||||
libeigen3-dev \
|
||||
libflann-dev \
|
||||
libfreeimage-dev \
|
||||
libgflags-dev \
|
||||
libglew-dev \
|
||||
@@ -34,15 +38,18 @@ RUN apt-get update && \
|
||||
libmetis-dev \
|
||||
libprotobuf-dev \
|
||||
libqt5opengl5-dev \
|
||||
libsqlite3-dev \
|
||||
libsuitesparse-dev \
|
||||
nano \
|
||||
protobuf-compiler \
|
||||
python3.10-dev \
|
||||
python3-pip \
|
||||
qtbase5-dev \
|
||||
sudo \
|
||||
wget && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
# Install GLOG (required by ceres).
|
||||
RUN git clone --branch v0.6.0 https://github.com/google/glog.git --single-branch && \
|
||||
cd glog && \
|
||||
@@ -69,22 +76,28 @@ RUN git clone --branch 2.1.0 https://ceres-solver.googlesource.com/ceres-solver.
|
||||
rm -rf ceres-solver
|
||||
|
||||
# Install colmap.
|
||||
RUN git clone --branch 3.7 https://github.com/colmap/colmap.git --single-branch && \
|
||||
RUN git clone --branch 3.8 https://github.com/colmap/colmap.git --single-branch && \
|
||||
cd colmap && \
|
||||
mkdir build && \
|
||||
cd build && \
|
||||
cmake .. -DCUDA_ENABLED=ON \
|
||||
-DCUDA_NVCC_FLAGS="--std c++14" && \
|
||||
-DCUDA_NVCC_FLAGS="--std c++14" \
|
||||
-DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES} && \
|
||||
make -j && \
|
||||
make install && \
|
||||
cd ../.. && \
|
||||
rm -rf colmap
|
||||
|
||||
# Create non root user and setup environment.
|
||||
RUN useradd -m -d /home/user -u 1000 user
|
||||
RUN useradd -m -d /home/user -g root -G sudo -u 1000 user
|
||||
RUN usermod -aG sudo user
|
||||
# Set user password
|
||||
RUN echo "user:user" | chpasswd
|
||||
# Ensure sudo group users are not asked for a password when using sudo command by ammending sudoers file
|
||||
RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
|
||||
|
||||
# Switch to new uer and workdir.
|
||||
USER 1000:1000
|
||||
USER 1000
|
||||
WORKDIR /home/user
|
||||
|
||||
# Add local user binary folder to PATH variable.
|
||||
@@ -93,16 +106,17 @@ SHELL ["/bin/bash", "-c"]
|
||||
|
||||
# Upgrade pip and install packages.
|
||||
RUN python3.10 -m pip install --upgrade pip setuptools pathtools promise
|
||||
# Install pytorch and submodules.
|
||||
RUN python3.10 -m pip install torch==1.12.1+cu116 torchvision==0.13.1+cu116 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu116
|
||||
# Install tynyCUDNN.
|
||||
# Install pytorch and submodules (Currently, we still use cu116 which is the latest version for toch 1.12.1 and is compatible with CUDA 11.8).
|
||||
RUN python3.10 -m pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116
|
||||
# Install tynyCUDNN (we need to set the target architectures as environment variable first).
|
||||
ENV TCNN_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES}
|
||||
RUN python3.10 -m pip install git+https://github.com/NVlabs/tiny-cuda-nn.git#subdirectory=bindings/torch
|
||||
|
||||
# Copy nerfstudio folder and give ownership to user.
|
||||
ADD . /home/user/nerfstudio
|
||||
USER root
|
||||
RUN chown -R user:user /home/user/nerfstudio
|
||||
USER 1000:1000
|
||||
RUN chown -R user /home/user/nerfstudio
|
||||
USER 1000
|
||||
|
||||
# Install nerfstudio dependencies.
|
||||
RUN cd nerfstudio && \
|
||||
|
||||
@@ -302,6 +302,8 @@ We provide the following support structures to make life easier for getting star
|
||||
|
||||
# Citation
|
||||
|
||||
You can find a paper writeup of the framework on [arXiv](https://arxiv.org/abs/2302.04264).
|
||||
|
||||
If you use this library or find the documentation useful for your research, please consider citing:
|
||||
|
||||
```
|
||||
|
||||
@@ -155,6 +155,8 @@ We'll be constantly growing this list! So make sure to check back in to see our
|
||||
|
||||
## Citation
|
||||
|
||||
You can find a paper writeup of the framework on [arXiv](https://arxiv.org/abs/2302.04264).
|
||||
|
||||
If you use this library or find the documentation useful for your research, please consider citing:
|
||||
|
||||
```none
|
||||
|
||||
@@ -92,8 +92,8 @@ pip install -e .[docs]
|
||||
## Use docker image
|
||||
Instead of installing and compiling prerequisites, setting up the environment and installing dependencies, a ready to use docker image is provided.
|
||||
### Prerequisites
|
||||
Docker ([get docker](https://docs.docker.com/get-docker/)) and nvidia GPU drivers ([get nvidia drivers](https://www.nvidia.de/Download/index.aspx?lang=de)), capable of working with CUDA 11.7, must be installed.
|
||||
The docker image can then either be pulled from [here](https://hub.docker.com/r/dromni/nerfstudio/tags) (replace <version> with the actual version, e.g. 0.1.10)
|
||||
Docker ([get docker](https://docs.docker.com/get-docker/)) and nvidia GPU drivers ([get nvidia drivers](https://www.nvidia.de/Download/index.aspx?lang=de)), capable of working with CUDA 11.8, must be installed.
|
||||
The docker image can then either be pulled from [here](https://hub.docker.com/r/dromni/nerfstudio/tags) (replace <version> with the actual version, e.g. 0.1.17)
|
||||
```bash
|
||||
docker pull dromni/nerfstudio:<version>
|
||||
```
|
||||
@@ -110,13 +110,15 @@ docker run --gpus all \ # Give the conta
|
||||
-p 7007:7007 \ # Map port from local machine to docker container (required to access the web interface/UI).
|
||||
--rm \ # Remove container after it is closed (recommended).
|
||||
-it \ # Start container in interactive mode.
|
||||
nerfstudio # Docker image name
|
||||
dromni/nerfstudio:<tag> # Docker image name if you pulled from docker hub.
|
||||
<--- OR --->
|
||||
nerfstudio # Docker image tag if you built the image from the Dockerfile by yourself using the command from above.
|
||||
```
|
||||
### Call nerfstudio commands directly
|
||||
Besides, the container can also directly be used by adding the nerfstudio command to the end.
|
||||
```bash
|
||||
docker run --gpus all -v /folder/of/your/data:/workspace/ -v /home/<YOUR_USER>/.cache/:/home/user/.cache/ -p 7007:7007 --rm -it # Parameters.
|
||||
nerfstudio \ # Docker image name
|
||||
dromni/nerfstudio:<tag> \ # Docker image name
|
||||
ns-process-data video --data /workspace/video.mp4 # Smaple command of nerfstudio.
|
||||
```
|
||||
### Note
|
||||
@@ -125,10 +127,10 @@ docker run --gpus all -v /folder/of/your/data:/workspace/ -v /home/<YOUR_USER>/.
|
||||
- Always use full paths, relative paths are known to create issues when being used in mounts into docker.
|
||||
- Everything inside the container, what is not in a mounted folder (workspace in the above example), will be permanently removed after destroying the container. Always do all your tasks and output folder in workdir!
|
||||
- The user inside the container is called 'user' and is mapped to the local user with ID 1000 (usually the first non-root user on Linux systems).
|
||||
- The container currently is based on nvidia/cuda:11.7.1-devel-ubuntu22.04, consequently it comes with CUDA 11.7 which must be supported by the nvidia driver. No local CUDA installation is required or will be affected by using the docker image.
|
||||
- The container currently is based on nvidia/cuda:11.8.0-devel-ubuntu22.04, consequently it comes with CUDA 11.8 which must be supported by the nvidia driver. No local CUDA installation is required or will be affected by using the docker image.
|
||||
- The docker image (respectively Ubuntu 22.04) comes with Python3.10, no older version of Python is installed.
|
||||
- If you call the container with commands directly, you still might want to add the interactive terminal ('-it') flag to get live log outputs of the nerfstudio scripts. In case the container is used in an automated environment the flag should be discarded.
|
||||
- The current version of docker is built for multi-architecture (CUDA architectures) use. The target architecture must be defined at build time for tinyCUDNN to be able to compile properly. If your GPU architecture is not covered by the following table you need to replace the number in the line ```ENV TCNN_CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37``` to you specific architecture. It also is a good idea to remove all architectures but yours (e.g. ```ENV TCNN_CUDA_ARCHITECTURES=86```) to speedup the docker build a lot.
|
||||
- The current version of docker is built for multi-architecture (CUDA architectures) use. The target architecture(s) must be defined at build time for Colmap and tinyCUDNN to be able to compile properly. If your GPU architecture is not covered by the following table you need to replace the number in the line ```ARG CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37``` to your specific architecture. It also is a good idea to remove all architectures but yours (e.g. ```ARG CUDA_ARCHITECTURES=86```) to speedup the docker build process a lot.
|
||||
|
||||
**Currently supported CUDA architectures in the docker image**
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ Here are the popular commands that we offer. If you've cloned the repo, you can
|
||||
| ns-process-data | Generate a dataset from your own data | scripts/process_data.py |
|
||||
| ns-download-data | Download existing captures | scripts/downloads/download_data.py |
|
||||
| ns-train | Generate a NeRF | scripts/train.py |
|
||||
| ns-viewer | View a trained NeRF | scripts/viewer/run_viewer.py |
|
||||
| ns-eval | Run evaluation metrics for your Model | scripts/eval.py |
|
||||
| ns-render | Render out a video of your NeRF | scripts/render.py |
|
||||
| ns-export | Export a NeRF into other formats | scripts/exporter.py |
|
||||
@@ -24,6 +25,7 @@ ns_process_data
|
||||
ns_download_data
|
||||
ns_train
|
||||
ns_render
|
||||
ns_viewer
|
||||
ns_export
|
||||
ns_eval
|
||||
```
|
||||
@@ -0,0 +1,9 @@
|
||||
# ns-viewer
|
||||
|
||||
```{eval-rst}
|
||||
.. argparse::
|
||||
:module: scripts.viewer.run_viewer
|
||||
:func: get_parser_fn
|
||||
:prog: ns-viewer
|
||||
:nodefault:
|
||||
```
|
||||
@@ -367,7 +367,7 @@ method_configs["dreamfusion"] = TrainerConfig(
|
||||
start_lambertian_training=1000,
|
||||
start_normals_training=500,
|
||||
opacity_loss_mult=0.001,
|
||||
prompting_type="location_based",
|
||||
positional_prompting="discrete",
|
||||
guidance_scale=100,
|
||||
),
|
||||
),
|
||||
|
||||
@@ -51,7 +51,11 @@ from nerfstudio.data.dataparsers.phototourism_dataparser import (
|
||||
)
|
||||
from nerfstudio.data.dataparsers.sdfstudio_dataparser import SDFStudioDataParserConfig
|
||||
from nerfstudio.data.datasets.base_dataset import InputDataset
|
||||
from nerfstudio.data.pixel_samplers import EquirectangularPixelSampler, PixelSampler
|
||||
from nerfstudio.data.pixel_samplers import (
|
||||
EquirectangularPixelSampler,
|
||||
PatchPixelSampler,
|
||||
PixelSampler,
|
||||
)
|
||||
from nerfstudio.data.utils.dataloaders import (
|
||||
CacheDataloader,
|
||||
FixedIndicesEvalDataloader,
|
||||
@@ -283,6 +287,8 @@ class VanillaDataManagerConfig(InstantiateConfig):
|
||||
"""The scale factor for scaling spatial data such as images, mask, semantics
|
||||
along with relevant information about camera intrinsics
|
||||
"""
|
||||
patch_size: int = 1
|
||||
"""Size of patch to sample from. If >1, patch-based sampling will be used."""
|
||||
|
||||
|
||||
class VanillaDataManager(DataManager): # pylint: disable=abstract-method
|
||||
@@ -346,6 +352,9 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method
|
||||
self, dataset: InputDataset, *args: Any, **kwargs: Any
|
||||
) -> PixelSampler:
|
||||
"""Infer pixel sampler to use."""
|
||||
if self.config.patch_size > 1:
|
||||
return PatchPixelSampler(*args, **kwargs, patch_size=self.config.patch_size)
|
||||
|
||||
# If all images are equirectangular, use equirectangular pixel sampler
|
||||
is_equirectangular = dataset.cameras.camera_type == CameraType.EQUIRECTANGULAR.value
|
||||
if is_equirectangular.all():
|
||||
|
||||
@@ -243,3 +243,54 @@ class EquirectangularPixelSampler(PixelSampler): # pylint: disable=too-few-publ
|
||||
).long()
|
||||
|
||||
return indices
|
||||
|
||||
|
||||
class PatchPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods
|
||||
"""Samples 'pixel_batch's from 'image_batch's. Samples square patches
|
||||
from the images randomly. Useful for patch-based losses.
|
||||
|
||||
Args:
|
||||
num_rays_per_batch: number of rays to sample per batch
|
||||
keep_full_image: whether or not to include a reference to the full image in returned batch
|
||||
patch_size: side length of patch. This must be consistent in the method
|
||||
config in order for samples to be reshaped into patches correctly.
|
||||
"""
|
||||
|
||||
def __init__(self, num_rays_per_batch: int, keep_full_image: bool = False, **kwargs) -> None:
|
||||
self.patch_size = kwargs["patch_size"]
|
||||
num_rays = (num_rays_per_batch // (self.patch_size**2)) * (self.patch_size**2)
|
||||
super().__init__(num_rays, keep_full_image, **kwargs)
|
||||
|
||||
# overrides base method
|
||||
def sample_method( # pylint: disable=no-self-use
|
||||
self,
|
||||
batch_size: int,
|
||||
num_images: int,
|
||||
image_height: int,
|
||||
image_width: int,
|
||||
mask: Optional[TensorType] = None,
|
||||
device: Union[torch.device, str] = "cpu",
|
||||
) -> TensorType["batch_size", 3]:
|
||||
|
||||
if mask:
|
||||
# Note: if there is a mask, sampling reduces back to uniform sampling
|
||||
indices = super().sample_method(batch_size, num_images, image_height, image_width, mask=mask, device=device)
|
||||
else:
|
||||
sub_bs = batch_size // (self.patch_size**2)
|
||||
indices = torch.rand((sub_bs, 3), device=device) * torch.tensor(
|
||||
[num_images, image_height - self.patch_size, image_width - self.patch_size],
|
||||
device=device,
|
||||
)
|
||||
|
||||
indices = indices.view(sub_bs, 1, 1, 3).broadcast_to(sub_bs, self.patch_size, self.patch_size, 3).clone()
|
||||
|
||||
yys, xxs = torch.meshgrid(
|
||||
torch.arange(self.patch_size, device=device), torch.arange(self.patch_size, device=device)
|
||||
)
|
||||
indices[:, ..., 1] += yys
|
||||
indices[:, ..., 2] += xxs
|
||||
|
||||
indices = torch.floor(indices).long()
|
||||
indices = indices.flatten(0, 2)
|
||||
|
||||
return indices
|
||||
@@ -146,7 +146,11 @@ class Trainer:
|
||||
'inference': does not load any dataset into memory
|
||||
"""
|
||||
self.pipeline = self.config.pipeline.setup(
|
||||
device=self.device, test_mode=test_mode, world_size=self.world_size, local_rank=self.local_rank, grad_scaler=self.grad_scaler
|
||||
device=self.device,
|
||||
test_mode=test_mode,
|
||||
world_size=self.world_size,
|
||||
local_rank=self.local_rank,
|
||||
grad_scaler=self.grad_scaler,
|
||||
)
|
||||
self.optimizers = self.setup_optimizers()
|
||||
|
||||
|
||||
@@ -151,6 +151,10 @@ def generate_point_cloud(
|
||||
CONSOLE.print(f"Please set --normal_output_name to one of: {outputs.keys()}", justify="center")
|
||||
sys.exit(1)
|
||||
normal = outputs[normal_output_name]
|
||||
assert (
|
||||
torch.min(normal) >= 0.0 and torch.max(normal) <= 1.0
|
||||
), "Normal values from method output must be in [0, 1]"
|
||||
normal = (normal * 2.0) - 1.0
|
||||
point = ray_bundle.origins + ray_bundle.directions * depth
|
||||
|
||||
if use_bounding_box:
|
||||
|
||||
@@ -29,7 +29,6 @@ from nerfstudio.data.scene_box import SceneBox
|
||||
from nerfstudio.field_components.activations import trunc_exp
|
||||
from nerfstudio.field_components.field_heads import FieldHeadNames, PredNormalsFieldHead
|
||||
from nerfstudio.fields.base_field import Field
|
||||
from nerfstudio.utils import math
|
||||
|
||||
try:
|
||||
import tinycudann as tcnn
|
||||
|
||||
@@ -33,27 +33,25 @@ from torch.cuda.amp import custom_bwd, custom_fwd
|
||||
from torch.cuda.amp.grad_scaler import GradScaler
|
||||
from torchtyping import TensorType
|
||||
|
||||
CONSOLE = Console(width=120)
|
||||
|
||||
try:
|
||||
from diffusers import PNDMScheduler, StableDiffusionPipeline
|
||||
from transformers import logging
|
||||
|
||||
except ImportError:
|
||||
CONSOLE.print("[bold red]Missing Stable Diffusion packages.")
|
||||
CONSOLE.print(r"Install using [yellow]pip install nerfstudio\[gen][/yellow]")
|
||||
CONSOLE.print(r"or [yellow]pip install -e .\[gen][/yellow] if installing from source.")
|
||||
sys.exit(1)
|
||||
SD_AVAILABLE = True
|
||||
|
||||
logging.set_verbosity_error()
|
||||
except ImportError:
|
||||
SD_AVAILABLE = False
|
||||
|
||||
CONSOLE = Console(width=120)
|
||||
IMG_DIM = 512
|
||||
CONST_SCALE = 0.18215
|
||||
SD_IDENTIFIERS = {
|
||||
'1-5':'runwayml/stable-diffusion-v1-5',
|
||||
'2-0':'stabilityai/stable-diffusion-2-base',
|
||||
'2-1':'stabilityai/stable-diffusion-2-1-base'
|
||||
"1-5": "runwayml/stable-diffusion-v1-5",
|
||||
"2-0": "stabilityai/stable-diffusion-2-base",
|
||||
"2-1": "stabilityai/stable-diffusion-2-1-base",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class UNet2DConditionOutput:
|
||||
"""Class to hold traced model"""
|
||||
@@ -88,9 +86,17 @@ class StableDiffusion(nn.Module):
|
||||
num_train_timesteps: number of training timesteps
|
||||
"""
|
||||
|
||||
def __init__(self, device: Union[torch.device, str], num_train_timesteps: int = 1000, version='1-5') -> None:
|
||||
def __init__(self, device: Union[torch.device, str], num_train_timesteps: int = 1000, version="1-5") -> None:
|
||||
super().__init__()
|
||||
|
||||
if not SD_AVAILABLE:
|
||||
CONSOLE.print("[bold red]Missing Stable Diffusion packages.")
|
||||
CONSOLE.print(r"Install using [yellow]pip install nerfstudio\[gen][/yellow]")
|
||||
CONSOLE.print(r"or [yellow]pip install -e .\[gen][/yellow] if installing from source.")
|
||||
sys.exit(1)
|
||||
|
||||
logging.set_verbosity_error()
|
||||
|
||||
self.device = device
|
||||
self.num_train_timesteps = num_train_timesteps
|
||||
|
||||
@@ -113,8 +119,8 @@ class StableDiffusion(nn.Module):
|
||||
pipe.enable_attention_slicing()
|
||||
|
||||
# use jitted unet
|
||||
filename_sd_id = sd_id.split('/')[-1]
|
||||
unet_traced_filename = Path(appdirs.user_data_dir("nerfstudio")) / f'{filename_sd_id}_unet_traced.pt'
|
||||
filename_sd_id = sd_id.split("/")[-1]
|
||||
unet_traced_filename = Path(appdirs.user_data_dir("nerfstudio")) / f"{filename_sd_id}_unet_traced.pt"
|
||||
if unet_traced_filename.exists():
|
||||
CONSOLE.print("Loading traced UNet.")
|
||||
unet_traced = torch.jit.load(unet_traced_filename)
|
||||
@@ -198,32 +204,34 @@ class StableDiffusion(nn.Module):
|
||||
Returns:
|
||||
The loss
|
||||
"""
|
||||
image = F.interpolate(image, (IMG_DIM, IMG_DIM), mode="bilinear")
|
||||
t = torch.randint(self.min_step, self.max_step + 1, [1], dtype=torch.long, device=self.device)
|
||||
latents = self.imgs_to_latent(image)
|
||||
# Disable autocast when using diffusers pipeline.
|
||||
with torch.autocast(device_type="cuda", enabled=False):
|
||||
image = F.interpolate(image.half(), (IMG_DIM, IMG_DIM), mode="bilinear")
|
||||
t = torch.randint(self.min_step, self.max_step + 1, [1], dtype=torch.long, device=self.device)
|
||||
latents = self.imgs_to_latent(image)
|
||||
|
||||
# predict the noise residual with unet, NO grad!
|
||||
with torch.no_grad():
|
||||
# add noise
|
||||
noise = torch.randn_like(latents)
|
||||
latents_noisy = self.scheduler.add_noise(latents, noise, t) # type: ignore
|
||||
# pred noise
|
||||
latent_model_input = torch.cat([latents_noisy] * 2)
|
||||
noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings).sample
|
||||
# predict the noise residual with unet, NO grad!
|
||||
with torch.no_grad():
|
||||
# add noise
|
||||
noise = torch.randn_like(latents)
|
||||
latents_noisy = self.scheduler.add_noise(latents, noise, t) # type: ignore
|
||||
# pred noise
|
||||
latent_model_input = torch.cat([latents_noisy] * 2)
|
||||
noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings).sample
|
||||
|
||||
# perform guidance
|
||||
noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
|
||||
noise_pred = noise_pred_text + guidance_scale * (noise_pred_text - noise_pred_uncond)
|
||||
# perform guidance
|
||||
noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
|
||||
noise_pred = noise_pred_text + guidance_scale * (noise_pred_text - noise_pred_uncond)
|
||||
|
||||
# w(t), sigma_t^2
|
||||
w = 1 - self.alphas[t]
|
||||
# w(t), sigma_t^2
|
||||
w = 1 - self.alphas[t]
|
||||
|
||||
grad = w * (noise_pred - noise)
|
||||
grad = torch.nan_to_num(grad)
|
||||
grad = w * (noise_pred - noise)
|
||||
grad = torch.nan_to_num(grad)
|
||||
|
||||
if grad_scaler is not None:
|
||||
latents = grad_scaler.scale(latents)
|
||||
loss = _SDSGradient.apply(latents, grad)
|
||||
if grad_scaler is not None:
|
||||
latents = grad_scaler.scale(latents)
|
||||
loss = _SDSGradient.apply(latents, grad)
|
||||
|
||||
return loss
|
||||
|
||||
|
||||
@@ -16,46 +16,95 @@
|
||||
Stable diffusion utils
|
||||
"""
|
||||
|
||||
def get_text_embedding(batch, prompting_type, text_embeddings, sd_device):
|
||||
""""""
|
||||
from torchtyping import TensorType
|
||||
from typing_extensions import Literal
|
||||
|
||||
base = text_embeddings["base_text_embedding"]
|
||||
front = text_embeddings["front_text_embedding"]
|
||||
side = text_embeddings["side_text_embedding"]
|
||||
top = text_embeddings["top_text_embedding"]
|
||||
back = text_embeddings["back_text_embedding"]
|
||||
from nerfstudio.generative.stable_diffusion import StableDiffusion
|
||||
|
||||
assert prompting_type in ("location_based", "interpolated", "base")
|
||||
|
||||
if prompting_type == "location_based":
|
||||
if batch["vertical"] < 40:
|
||||
text_embedding = top
|
||||
elif batch["central"] > 315 or batch["central"] <= 45:
|
||||
text_embedding = front
|
||||
elif batch["central"] > 45 and batch["central"] <= 135:
|
||||
text_embedding = side
|
||||
elif batch["central"] > 135 and batch["central"] <= 225:
|
||||
text_embedding = back
|
||||
else: # batch["central"] > 225 and batch["central"] <= 315:
|
||||
text_embedding = side
|
||||
class PositionalTextEmbeddings:
|
||||
"""Postional Prompts. Currently the following location based text embeddings are supported:
|
||||
- discrete: Choose the embedding closest to the camera position
|
||||
- interpolated: Interpolate between the embeddings based on the camera position
|
||||
- off: Don't modify the text embedding based on the camera position
|
||||
|
||||
elif prompting_type == "interpolated":
|
||||
horiz = batch["central"].to(sd_device)
|
||||
vert = max(batch["vertical"].to(sd_device), 0)
|
||||
Args:
|
||||
base_prompt: Prompt for base view
|
||||
top_prompt: Prompt for top view
|
||||
side_prompt: Prompt for side view
|
||||
back_prompt: Prompt for back view
|
||||
front_prompt: Prompt for front view
|
||||
stable_diffusion: Instance of StableDiffusion
|
||||
positional_prompting: how to incorporate position into prompt.
|
||||
"""
|
||||
|
||||
if batch["central"] > 0 or batch["central"] <= 90:
|
||||
text_embedding = (horiz) * side + (90 - horiz) * front
|
||||
elif batch["central"] > 90 and batch["central"] <= 180:
|
||||
text_embedding = (horiz - 90) * back + (180 - horiz) * side
|
||||
elif batch["central"] > 180 and batch["central"] <= 270:
|
||||
text_embedding = (horiz - 180) * side + (270 - horiz) * back
|
||||
else: # batch["central"] > 270 and batch["central"] <= 360:
|
||||
text_embedding = (horiz - 270) * front + (360 - horiz) * side
|
||||
|
||||
text_embedding = text_embedding / 90.0
|
||||
text_embedding = (vert * text_embedding + (90 - vert) * top) / 90.0
|
||||
def __init__(
|
||||
self,
|
||||
base_prompt: str,
|
||||
top_prompt: str,
|
||||
side_prompt: str,
|
||||
back_prompt: str,
|
||||
front_prompt: str,
|
||||
stable_diffusion: StableDiffusion,
|
||||
positional_prompting: Literal["discrete", "interpolated", "off"] = "discrete",
|
||||
):
|
||||
self.positional_prompting = positional_prompting
|
||||
self.sd_device = stable_diffusion.device
|
||||
self.sd = stable_diffusion
|
||||
self.update_prompt(base_prompt, top_prompt, side_prompt, back_prompt, front_prompt)
|
||||
|
||||
elif prompting_type == "base":
|
||||
text_embedding = base
|
||||
def update_prompt(self, base_prompt: str, top_prompt: str, side_prompt: str, back_prompt: str, front_prompt: str):
|
||||
"""Update the text embeddings based on the new prompts.
|
||||
|
||||
return text_embedding
|
||||
Args:
|
||||
base_prompt: Prompt for base view
|
||||
top_prompt: Prompt for top view
|
||||
side_prompt: Prompt for side view
|
||||
back_prompt: Prompt for back view
|
||||
front_prompt: Prompt for front view
|
||||
"""
|
||||
self.base_embed = self.sd.get_text_embeds(base_prompt, "")
|
||||
self.top_embed = self.sd.get_text_embeds(top_prompt, "")
|
||||
self.side_embed = self.sd.get_text_embeds(side_prompt, "")
|
||||
self.back_embed = self.sd.get_text_embeds(back_prompt, "")
|
||||
self.front_embed = self.sd.get_text_embeds(front_prompt, "")
|
||||
|
||||
def get_text_embedding(self, vertical_angle: TensorType[1], horizonal_angle: TensorType[1]):
|
||||
"""Get text embedding based on the position of the camera relative to the scene.
|
||||
This trick is used in Dreamfusion (https://dreamfusion3d.github.io/).
|
||||
|
||||
Args:
|
||||
vertical_angle: vertical angle of the camera
|
||||
horizonal_angle: horizonal angle of the camera
|
||||
"""
|
||||
|
||||
if self.positional_prompting == "discrete":
|
||||
if vertical_angle < 40:
|
||||
text_embedding = self.top_embed
|
||||
elif 315 < horizonal_angle <= 45:
|
||||
text_embedding = self.front_embed
|
||||
elif 45 < horizonal_angle <= 135:
|
||||
text_embedding = self.side_embed
|
||||
elif 135 < horizonal_angle <= 225:
|
||||
text_embedding = self.back_embed
|
||||
else: # horizonal_angle > 225 and horizonal_angle <= 315:
|
||||
text_embedding = self.side_embed
|
||||
elif self.positional_prompting == "interpolated":
|
||||
horiz = horizonal_angle.to(self.sd_device)
|
||||
vert = max(vertical_angle.to(self.sd_device), 0)
|
||||
|
||||
if 0 < horizonal_angle <= 90:
|
||||
text_embedding = (horiz) * self.side_embed + (90 - horiz) * self.front_embed
|
||||
elif 90 < horizonal_angle <= 180:
|
||||
text_embedding = (horiz - 90) * self.back_embed + (180 - horiz) * self.side_embed
|
||||
elif 180 < horizonal_angle <= 270:
|
||||
text_embedding = (horiz - 180) * self.side_embed + (270 - horiz) * self.back_embed
|
||||
else: # 270 < horizonal_angle <= 360:
|
||||
text_embedding = (horiz - 270) * self.front_embed + (360 - horiz) * self.side_embed
|
||||
|
||||
text_embedding = text_embedding / 90.0
|
||||
text_embedding = (vert * text_embedding + (90 - vert) * self.top_embed) / 90.0
|
||||
else:
|
||||
text_embedding = self.base_embed
|
||||
|
||||
return text_embedding
|
||||
@@ -20,8 +20,10 @@ from enum import Enum
|
||||
import torch
|
||||
from torch import nn
|
||||
from torchtyping import TensorType
|
||||
from typing_extensions import Literal
|
||||
|
||||
from nerfstudio.cameras.rays import RaySamples
|
||||
from nerfstudio.utils.math import masked_reduction
|
||||
|
||||
L1Loss = nn.L1Loss
|
||||
MSELoss = nn.MSELoss
|
||||
@@ -308,3 +310,218 @@ def depth_loss(
|
||||
return urban_radiance_field_depth_loss(weights, termination_depth, predicted_depth, steps, sigma)
|
||||
|
||||
raise NotImplementedError("Provided depth loss type not implemented.")
|
||||
|
||||
|
||||
def monosdf_normal_loss(
|
||||
normal_pred: TensorType["num_samples", 3], normal_gt: TensorType["num_samples", 3]
|
||||
) -> TensorType[0]:
|
||||
"""
|
||||
Normal consistency loss proposed in monosdf - https://niujinshuchong.github.io/monosdf/
|
||||
Enforces consistency between the volume rendered normal and the predicted monocular normal.
|
||||
With both angluar and L1 loss. Eq 14 https://arxiv.org/pdf/2206.00665.pdf
|
||||
Args:
|
||||
normal_pred: volume rendered normal
|
||||
normal_gt: monocular normal
|
||||
"""
|
||||
normal_gt = torch.nn.functional.normalize(normal_gt, p=2, dim=-1)
|
||||
normal_pred = torch.nn.functional.normalize(normal_pred, p=2, dim=-1)
|
||||
l1 = torch.abs(normal_pred - normal_gt).sum(dim=-1).mean()
|
||||
cos = (1.0 - torch.sum(normal_pred * normal_gt, dim=-1)).mean()
|
||||
return l1 + cos
|
||||
|
||||
|
||||
class MiDaSMSELoss(nn.Module):
|
||||
"""
|
||||
data term from MiDaS paper
|
||||
"""
|
||||
|
||||
def __init__(self, reduction_type: Literal["image", "batch"] = "batch"):
|
||||
super().__init__()
|
||||
|
||||
self.reduction_type = reduction_type
|
||||
# reduction here is different from the image/batch-based reduction. This is either "mean" or "sum"
|
||||
self.mse_loss = MSELoss(reduction="none")
|
||||
|
||||
def forward(
|
||||
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
|
||||
) -> TensorType[0]:
|
||||
"""
|
||||
Args:
|
||||
prediction: predicted depth map
|
||||
target: ground truth depth map
|
||||
mask: mask of valid pixels
|
||||
Returns:
|
||||
mse loss based on reduction function
|
||||
"""
|
||||
summed_mask = torch.sum(mask, (1, 2))
|
||||
image_loss = torch.sum(self.mse_loss(prediction, target) * mask, (1, 2))
|
||||
image_loss = masked_reduction(image_loss, summed_mask, self.reduction_type)
|
||||
|
||||
return image_loss
|
||||
|
||||
|
||||
# losses based on https://github.com/autonomousvision/monosdf/blob/main/code/model/loss.py
|
||||
class GradientLoss(nn.Module):
|
||||
"""
|
||||
multiscale, scale-invariant gradient matching term to the disparity space.
|
||||
This term biases discontinuities to be sharp and to coincide with discontinuities in the ground truth
|
||||
More info here https://arxiv.org/pdf/1907.01341.pdf Equation 11
|
||||
"""
|
||||
|
||||
def __init__(self, scales: int = 4, reduction_type: Literal["image", "batch"] = "batch"):
|
||||
"""
|
||||
Args:
|
||||
scales: number of scales to use
|
||||
reduction_type: either "batch" or "image"
|
||||
"""
|
||||
super().__init__()
|
||||
self.reduction_type = reduction_type
|
||||
self.__scales = scales
|
||||
|
||||
def forward(
|
||||
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
|
||||
) -> TensorType[0]:
|
||||
"""
|
||||
Args:
|
||||
prediction: predicted depth map
|
||||
target: ground truth depth map
|
||||
mask: mask of valid pixels
|
||||
Returns:
|
||||
gradient loss based on reduction function
|
||||
"""
|
||||
total = 0
|
||||
|
||||
for scale in range(self.__scales):
|
||||
step = pow(2, scale)
|
||||
|
||||
grad_loss = self.gradient_loss(
|
||||
prediction[:, ::step, ::step],
|
||||
target[:, ::step, ::step],
|
||||
mask[:, ::step, ::step],
|
||||
)
|
||||
total += grad_loss
|
||||
|
||||
return total
|
||||
|
||||
def gradient_loss(
|
||||
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
|
||||
) -> TensorType[0]:
|
||||
"""
|
||||
multiscale, scale-invariant gradient matching term to the disparity space.
|
||||
This term biases discontinuities to be sharp and to coincide with discontinuities in the ground truth
|
||||
More info here https://arxiv.org/pdf/1907.01341.pdf Equation 11
|
||||
Args:
|
||||
prediction: predicted depth map
|
||||
target: ground truth depth map
|
||||
reduction: reduction function, either reduction_batch_based or reduction_image_based
|
||||
Returns:
|
||||
gradient loss based on reduction function
|
||||
"""
|
||||
summed_mask = torch.sum(mask, (1, 2))
|
||||
diff = prediction - target
|
||||
diff = torch.mul(mask, diff)
|
||||
|
||||
grad_x = torch.abs(diff[:, :, 1:] - diff[:, :, :-1])
|
||||
mask_x = torch.mul(mask[:, :, 1:], mask[:, :, :-1])
|
||||
grad_x = torch.mul(mask_x, grad_x)
|
||||
|
||||
grad_y = torch.abs(diff[:, 1:, :] - diff[:, :-1, :])
|
||||
mask_y = torch.mul(mask[:, 1:, :], mask[:, :-1, :])
|
||||
grad_y = torch.mul(mask_y, grad_y)
|
||||
|
||||
image_loss = torch.sum(grad_x, (1, 2)) + torch.sum(grad_y, (1, 2))
|
||||
image_loss = masked_reduction(image_loss, summed_mask, self.reduction_type)
|
||||
|
||||
return image_loss
|
||||
|
||||
|
||||
class ScaleAndShiftInvariantLoss(nn.Module):
|
||||
"""
|
||||
Scale and shift invariant loss as described in
|
||||
"Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer"
|
||||
https://arxiv.org/pdf/1907.01341.pdf
|
||||
"""
|
||||
|
||||
def __init__(self, alpha: float = 0.5, scales: int = 4, reduction_type: Literal["image", "batch"] = "batch"):
|
||||
"""
|
||||
Args:
|
||||
alpha: weight of the regularization term
|
||||
scales: number of scales to use
|
||||
reduction_type: either "batch" or "image"
|
||||
"""
|
||||
super().__init__()
|
||||
self.__data_loss = MiDaSMSELoss(reduction_type=reduction_type)
|
||||
self.__regularization_loss = GradientLoss(scales=scales, reduction_type=reduction_type)
|
||||
self.__alpha = alpha
|
||||
|
||||
self.__prediction_ssi = None
|
||||
|
||||
@classmethod
|
||||
def compute_scale_and_shift(
|
||||
cls, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
|
||||
):
|
||||
"""
|
||||
More info here: https://arxiv.org/pdf/2206.00665.pdf supplementary section A2 Depth Consistency Loss
|
||||
This function computes scale/shift required to normalizes predicted depth map,
|
||||
to allow for using normalized depth maps as input from monocular depth estimation networks.
|
||||
These networks are trained such that they predict normalized depth maps.
|
||||
|
||||
Solves for scale/shift using a least squares approach with a closed form solution:
|
||||
Based on:
|
||||
https://github.com/autonomousvision/monosdf/blob/d9619e948bf3d85c6adec1a643f679e2e8e84d4b/code/model/loss.py#L7
|
||||
Args:
|
||||
prediction: predicted depth map
|
||||
target: ground truth depth map
|
||||
mask: mask of valid pixels
|
||||
Returns:
|
||||
scale and shift for depth prediction
|
||||
"""
|
||||
# system matrix: A = [[a_00, a_01], [a_10, a_11]]
|
||||
a_00 = torch.sum(mask * prediction * prediction, (1, 2))
|
||||
a_01 = torch.sum(mask * prediction, (1, 2))
|
||||
a_11 = torch.sum(mask, (1, 2))
|
||||
|
||||
# right hand side: b = [b_0, b_1]
|
||||
b_0 = torch.sum(mask * prediction * target, (1, 2))
|
||||
b_1 = torch.sum(mask * target, (1, 2))
|
||||
|
||||
# solution: x = A^-1 . b = [[a_11, -a_01], [-a_10, a_00]] / (a_00 * a_11 - a_01 * a_10) . b
|
||||
scale = torch.zeros_like(b_0)
|
||||
shift = torch.zeros_like(b_1)
|
||||
|
||||
det = a_00 * a_11 - a_01 * a_01
|
||||
valid = det.nonzero()
|
||||
|
||||
scale[valid] = (a_11[valid] * b_0[valid] - a_01[valid] * b_1[valid]) / det[valid]
|
||||
shift[valid] = (-a_01[valid] * b_0[valid] + a_00[valid] * b_1[valid]) / det[valid]
|
||||
|
||||
return scale, shift
|
||||
|
||||
def forward(
|
||||
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
|
||||
) -> TensorType[0]:
|
||||
"""
|
||||
Args:
|
||||
prediction: predicted depth map (unnormalized)
|
||||
target: ground truth depth map (normalized)
|
||||
mask: mask of valid pixels
|
||||
Returns:
|
||||
scale and shift invariant loss
|
||||
"""
|
||||
scale, shift = ScaleAndShiftInvariantLoss.compute_scale_and_shift(prediction, target, mask)
|
||||
self.__prediction_ssi = scale.view(-1, 1, 1) * prediction + shift.view(-1, 1, 1)
|
||||
|
||||
total = self.__data_loss(self.__prediction_ssi, target, mask)
|
||||
if self.__alpha > 0:
|
||||
total += self.__alpha * self.__regularization_loss(self.__prediction_ssi, target, mask)
|
||||
|
||||
return total
|
||||
|
||||
def __get_prediction_ssi(self):
|
||||
"""
|
||||
scale and shift invariant prediction
|
||||
from https://arxiv.org/pdf/1907.01341.pdf equation 1
|
||||
"""
|
||||
return self.__prediction_ssi
|
||||
|
||||
prediction_ssi = property(__get_prediction_ssi)
|
||||
@@ -37,6 +37,7 @@ from torchtyping import TensorType
|
||||
from typing_extensions import Literal
|
||||
|
||||
from nerfstudio.cameras.rays import RaySamples
|
||||
from nerfstudio.utils import colors
|
||||
from nerfstudio.utils.math import components_from_spherical_harmonics, safe_normalize
|
||||
|
||||
BACKGROUND_COLOR_OVERRIDE: Optional[TensorType[3]] = None
|
||||
@@ -70,7 +71,7 @@ class RGBRenderer(nn.Module):
|
||||
cls,
|
||||
rgb: TensorType["bs":..., "num_samples", 3],
|
||||
weights: TensorType["bs":..., "num_samples", 1],
|
||||
background_color: Union[Literal["random", "black", "last_sample"], TensorType[3]] = "random",
|
||||
background_color: Union[Literal["random", "white", "black", "last_sample"], TensorType[3]] = "random",
|
||||
ray_indices: Optional[TensorType["num_samples"]] = None,
|
||||
num_rays: Optional[int] = None,
|
||||
) -> TensorType["bs":..., 3]:
|
||||
@@ -102,8 +103,8 @@ class RGBRenderer(nn.Module):
|
||||
background_color = rgb[..., -1, :]
|
||||
if background_color == "random":
|
||||
background_color = torch.rand_like(comp_rgb).to(rgb.device)
|
||||
if background_color == "black":
|
||||
background_color = torch.zeros_like(comp_rgb).to(rgb.device)
|
||||
if isinstance(background_color, str) and background_color in colors.COLORS_DICT:
|
||||
background_color = colors.COLORS_DICT[background_color].to(rgb.device)
|
||||
|
||||
assert isinstance(background_color, torch.Tensor)
|
||||
comp_rgb = comp_rgb + background_color.to(weights.device) * (1.0 - accumulated_weight)
|
||||
|
||||
@@ -70,7 +70,8 @@ class DepthNerfactoModel(NerfactoModel):
|
||||
|
||||
def get_outputs(self, ray_bundle: RayBundle):
|
||||
outputs = super().get_outputs(ray_bundle)
|
||||
outputs["directions_norm"] = ray_bundle.metadata["directions_norm"]
|
||||
if ray_bundle.metadata is not None and "directions_norm" in ray_bundle.metadata:
|
||||
outputs["directions_norm"] = ray_bundle.metadata["directions_norm"]
|
||||
return outputs
|
||||
|
||||
def get_metrics_dict(self, outputs, batch):
|
||||
|
||||
@@ -19,12 +19,12 @@ DreamFusion implementation.
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Tuple, Type, Optional
|
||||
from typing import Dict, List, Optional, Tuple, Type
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch.nn import Parameter
|
||||
from torch.cuda.amp.grad_scaler import GradScaler
|
||||
from typing_extensions import Literal
|
||||
|
||||
from nerfstudio.cameras.rays import RayBundle
|
||||
from nerfstudio.engine.callbacks import (
|
||||
@@ -32,11 +32,11 @@ from nerfstudio.engine.callbacks import (
|
||||
TrainingCallbackAttributes,
|
||||
TrainingCallbackLocation,
|
||||
)
|
||||
from nerfstudio.generative.stable_diffusion import StableDiffusion
|
||||
from nerfstudio.generative.stable_diffusion_utils import get_text_embedding
|
||||
from nerfstudio.field_components.field_heads import FieldHeadNames
|
||||
from nerfstudio.fields.density_fields import HashMLPDensityField
|
||||
from nerfstudio.fields.dreamfusion_field import DreamFusionField
|
||||
from nerfstudio.generative.stable_diffusion import StableDiffusion
|
||||
from nerfstudio.generative.stable_diffusion_utils import PositionalTextEmbeddings
|
||||
from nerfstudio.model_components.losses import (
|
||||
MSELoss,
|
||||
distortion_loss,
|
||||
@@ -77,7 +77,7 @@ class DreamFusionModelConfig(ModelConfig):
|
||||
"""Randomizes light source per output."""
|
||||
initialize_density: bool = True
|
||||
"""Initialize density in center of scene."""
|
||||
init_density_strength: float = 0.1
|
||||
init_density_strength: float = 10
|
||||
"""Initial strength of center density"""
|
||||
sphere_collider: bool = True
|
||||
"""Use spherical collider instead of box"""
|
||||
@@ -135,8 +135,8 @@ class DreamFusionModelConfig(ModelConfig):
|
||||
"""enables location based prompting"""
|
||||
interpolated_prompting: bool = False
|
||||
"""enables interpolated location prompting"""
|
||||
prompting_type: str = "location_based"
|
||||
"""choose between location_based, interpolated, base"""
|
||||
positional_prompting: Literal["discrete", "interpolated", "base"] = "discrete"
|
||||
""" how to incorporate position into prompt"""
|
||||
top_prompt: str = ", overhead view"
|
||||
"""appended to prompt for overhead view"""
|
||||
side_prompt: str = ", side view"
|
||||
@@ -149,7 +149,8 @@ class DreamFusionModelConfig(ModelConfig):
|
||||
"""guidance scale for sds loss"""
|
||||
stablediffusion_device: Optional[str] = None
|
||||
"""device for stable diffusion"""
|
||||
sd_version: str = '1-5'
|
||||
sd_version: str = "1-5"
|
||||
|
||||
|
||||
class DreamFusionModel(Model):
|
||||
"""DreamFusionModel Model
|
||||
@@ -174,17 +175,17 @@ class DreamFusionModel(Model):
|
||||
self.random_background = config.random_background
|
||||
self.density_strength = 1.0
|
||||
self.target_transmittance = config.target_transmittance_start
|
||||
self.grad_scaler = kwargs['grad_scaler']
|
||||
self.grad_scaler = kwargs["grad_scaler"]
|
||||
|
||||
self.prompting_type = config.prompting_type
|
||||
self.positional_prompting = config.positional_prompting
|
||||
self.guidance_scale = config.guidance_scale
|
||||
self.top_prompt = config.top_prompt
|
||||
self.side_prompt = config.side_prompt
|
||||
self.back_prompt = config.back_prompt
|
||||
self.back_prompt = config.back_prompt
|
||||
self.front_prompt = config.front_prompt
|
||||
|
||||
self.sd_device = (
|
||||
torch.device(kwargs['device'])
|
||||
torch.device(kwargs["device"])
|
||||
if config.stablediffusion_device is None
|
||||
else torch.device(config.stablediffusion_device)
|
||||
)
|
||||
@@ -196,13 +197,15 @@ class DreamFusionModel(Model):
|
||||
super().populate_modules()
|
||||
|
||||
self.sd = StableDiffusion(self.sd_device, version=self.sd_version)
|
||||
self.text_embeddings = {
|
||||
"top_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.top_prompt}", ""),
|
||||
"front_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.front_prompt}", ""),
|
||||
"side_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.side_prompt}", ""),
|
||||
"back_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.back_prompt}", ""),
|
||||
"base_text_embedding": self.sd.get_text_embeds(self.cur_prompt, "")
|
||||
}
|
||||
self.text_embeddings = PositionalTextEmbeddings(
|
||||
base_prompt=self.cur_prompt,
|
||||
top_prompt=self.cur_prompt + self.top_prompt,
|
||||
side_prompt=self.cur_prompt + self.side_prompt,
|
||||
back_prompt=self.cur_prompt + self.back_prompt,
|
||||
front_prompt=self.cur_prompt + self.front_prompt,
|
||||
stable_diffusion=self.sd,
|
||||
positional_prompting=self.positional_prompting,
|
||||
)
|
||||
|
||||
# setting up fields
|
||||
self.field = DreamFusionField(self.scene_box.aabb, max_res=self.config.max_res)
|
||||
@@ -339,25 +342,26 @@ class DreamFusionModel(Model):
|
||||
param_groups["fields"] = list(self.field.parameters())
|
||||
return param_groups
|
||||
|
||||
def get_outputs(self, ray_bundle: RayBundle):
|
||||
def get_outputs(self, ray_bundle: RayBundle): # pylint: disable=too-many-statements
|
||||
# uniform sampling
|
||||
background_rgb = self.field.get_background_rgb(ray_bundle)
|
||||
ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns)
|
||||
field_outputs = self.field(ray_samples, compute_normals=True)
|
||||
weights = ray_samples.get_weights(field_outputs[FieldHeadNames.DENSITY])
|
||||
weights_list.append(weights)
|
||||
ray_samples_list.append(ray_samples)
|
||||
density = field_outputs[FieldHeadNames.DENSITY]
|
||||
|
||||
if self.initialize_density and self.training:
|
||||
if self.initialize_density:
|
||||
pos = ray_samples.frustums.get_positions()
|
||||
density_blob = (
|
||||
self.config.init_density_strength
|
||||
* self.density_strength
|
||||
* torch.exp(-torch.norm(pos, dim=-1) / (2 * 0.04))[..., None]
|
||||
* torch.exp(-torch.norm(pos, dim=-1) * 10)[..., None]
|
||||
)
|
||||
density += density_blob
|
||||
|
||||
weights = ray_samples.get_weights(density)
|
||||
weights_list.append(weights)
|
||||
ray_samples_list.append(ray_samples)
|
||||
|
||||
accumulation = self.renderer_accumulation(weights)
|
||||
depth = self.renderer_depth(weights, ray_samples)
|
||||
rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights)
|
||||
@@ -421,7 +425,7 @@ class DreamFusionModel(Model):
|
||||
else:
|
||||
outputs["train_output"] = outputs["rgb"]
|
||||
|
||||
if self.training or True:
|
||||
if self.training:
|
||||
outputs["rendered_orientation_loss"] = orientation_loss(
|
||||
weights.detach(), field_outputs[FieldHeadNames.NORMALS], ray_bundle.directions
|
||||
)
|
||||
@@ -463,28 +467,30 @@ class DreamFusionModel(Model):
|
||||
|
||||
if self.prompt != self.cur_prompt:
|
||||
self.cur_prompt = self.prompt
|
||||
self.text_embeddings = {
|
||||
"top_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.top_prompt}", ""),
|
||||
"front_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.front_prompt}", ""),
|
||||
"side_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.side_prompt}", ""),
|
||||
"back_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.back_prompt}", ""),
|
||||
"base_text_embedding": self.sd.get_text_embeds(self.cur_prompt, "")
|
||||
}
|
||||
|
||||
text_embedding = get_text_embedding(batch, self.prompting_type, self.text_embeddings, self.sd_device)
|
||||
|
||||
self.text_embeddings.update_prompt(
|
||||
base_prompt=self.cur_prompt,
|
||||
top_prompt=self.cur_prompt + self.top_prompt,
|
||||
side_prompt=self.cur_prompt + self.side_prompt,
|
||||
back_prompt=self.cur_prompt + self.back_prompt,
|
||||
front_prompt=self.cur_prompt + self.front_prompt,
|
||||
)
|
||||
|
||||
text_embedding = self.text_embeddings.get_text_embedding(
|
||||
vertical_angle=batch["vertical"], horizonal_angle=batch["central"]
|
||||
)
|
||||
|
||||
train_output = (
|
||||
outputs["train_output"].view(1, int(outputs["train_output"].shape[0] ** 0.5), int(outputs["train_output"].shape[0] ** 0.5), 3)
|
||||
outputs["train_output"]
|
||||
.view(1, int(outputs["train_output"].shape[0] ** 0.5), int(outputs["train_output"].shape[0] ** 0.5), 3)
|
||||
.permute(0, 3, 1, 2)
|
||||
)
|
||||
|
||||
with torch.autocast(device_type="cuda", dtype=torch.float16):
|
||||
sds_loss = self.sd.sds_loss(
|
||||
text_embedding.to(self.sd_device),
|
||||
train_output.to(self.sd_device),
|
||||
guidance_scale=int(self.guidance_scale),
|
||||
grad_scaler=self.grad_scaler,
|
||||
)
|
||||
sds_loss = self.sd.sds_loss(
|
||||
text_embedding.to(self.sd_device),
|
||||
train_output.to(self.sd_device),
|
||||
guidance_scale=int(self.guidance_scale),
|
||||
grad_scaler=self.grad_scaler,
|
||||
)
|
||||
|
||||
loss_dict["sds_loss"] = sds_loss.to(self.device)
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ from nerfstudio.model_components.renderers import (
|
||||
RGBRenderer,
|
||||
)
|
||||
from nerfstudio.models.base_model import Model, ModelConfig
|
||||
from nerfstudio.utils import colormaps, colors
|
||||
from nerfstudio.utils import colormaps
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -123,11 +123,7 @@ class NGPModel(Model):
|
||||
)
|
||||
|
||||
# renderers
|
||||
background_color = "random"
|
||||
if self.config.background_color in ["white", "black"]:
|
||||
background_color = colors.COLORS_DICT[self.config.background_color]
|
||||
|
||||
self.renderer_rgb = RGBRenderer(background_color=background_color)
|
||||
self.renderer_rgb = RGBRenderer(background_color=self.config.background_color)
|
||||
self.renderer_accumulation = AccumulationRenderer()
|
||||
self.renderer_depth = DepthRenderer(method="expected")
|
||||
|
||||
|
||||
@@ -68,7 +68,7 @@ class NerfactoModelConfig(ModelConfig):
|
||||
"""How far along the ray to start sampling."""
|
||||
far_plane: float = 1000.0
|
||||
"""How far along the ray to stop sampling."""
|
||||
background_color: Literal["random", "last_sample"] = "last_sample"
|
||||
background_color: Literal["random", "last_sample", "black", "white"] = "last_sample"
|
||||
"""Whether to randomize the background color."""
|
||||
num_levels: int = 16
|
||||
"""Number of levels of the hashmap for the base mlp."""
|
||||
|
||||
@@ -50,6 +50,7 @@ from nerfstudio.model_components.renderers import (
|
||||
RGBRenderer,
|
||||
)
|
||||
from nerfstudio.model_components.scene_colliders import NearFarCollider
|
||||
from nerfstudio.model_components.shaders import NormalsShader
|
||||
from nerfstudio.models.base_model import Model
|
||||
from nerfstudio.models.nerfacto import NerfactoModel, NerfactoModelConfig
|
||||
|
||||
@@ -63,7 +64,7 @@ class NerfplayerNerfactoModelConfig(NerfactoModelConfig):
|
||||
"""How far along the ray to start sampling."""
|
||||
far_plane: float = 1000.0
|
||||
"""How far along the ray to stop sampling."""
|
||||
background_color: Literal["random", "last_sample"] = "random"
|
||||
background_color: Literal["random", "last_sample", "black", "white"] = "random"
|
||||
"""Whether to randomize the background color. (Random is reported to be better on DyCheck.)"""
|
||||
num_levels: int = 16
|
||||
"""Hashing grid parameter."""
|
||||
@@ -162,6 +163,9 @@ class NerfplayerNerfactoModel(NerfactoModel):
|
||||
self.renderer_depth = DepthRenderer(method="expected") # for depth loss
|
||||
self.renderer_normals = NormalsRenderer()
|
||||
|
||||
# shaders
|
||||
self.normals_shader = NormalsShader()
|
||||
|
||||
# losses
|
||||
self.rgb_loss = MSELoss()
|
||||
|
||||
@@ -192,8 +196,12 @@ class NerfplayerNerfactoModel(NerfactoModel):
|
||||
}
|
||||
|
||||
if self.config.predict_normals:
|
||||
outputs["normals"] = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights)
|
||||
outputs["pred_normals"] = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights)
|
||||
outputs["normals"] = self.normals_shader(
|
||||
self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights)
|
||||
)
|
||||
outputs["pred_normals"] = self.normals_shader(
|
||||
self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights)
|
||||
)
|
||||
|
||||
# These use a lot of GPU memory, so we avoid storing them for eval.
|
||||
if self.training:
|
||||
|
||||
@@ -42,7 +42,6 @@ from nerfstudio.model_components.renderers import (
|
||||
)
|
||||
from nerfstudio.models.base_model import Model
|
||||
from nerfstudio.models.instant_ngp import InstantNGPModelConfig, NGPModel
|
||||
from nerfstudio.utils import colors
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -128,13 +127,6 @@ class NerfplayerNGPModel(NGPModel):
|
||||
) # need to update the density_fn later during forward (for input time)
|
||||
|
||||
# renderers
|
||||
self.train_background_color = self.config.train_background_color
|
||||
self.eval_background_color = self.config.eval_background_color
|
||||
if self.config.train_background_color in ["white", "black"]:
|
||||
self.train_background_color = colors.COLORS_DICT[self.config.train_background_color]
|
||||
if self.config.eval_background_color in ["white", "black"]:
|
||||
self.eval_background_color = colors.COLORS_DICT[self.config.eval_background_color]
|
||||
|
||||
self.renderer_rgb = RGBRenderer() # will update bgcolor later during forward
|
||||
self.renderer_accumulation = AccumulationRenderer()
|
||||
self.renderer_depth = DepthRenderer(method="expected")
|
||||
@@ -175,9 +167,9 @@ class NerfplayerNGPModel(NGPModel):
|
||||
|
||||
# update bgcolor in the renderer; usually random color for training and fixed color for inference
|
||||
if self.training:
|
||||
self.renderer_rgb.background_color = self.train_background_color
|
||||
self.renderer_rgb.background_color = self.config.train_background_color
|
||||
else:
|
||||
self.renderer_rgb.background_color = self.eval_background_color
|
||||
self.renderer_rgb.background_color = self.config.eval_background_color
|
||||
rgb = self.renderer_rgb(
|
||||
rgb=field_outputs[FieldHeadNames.RGB],
|
||||
weights=weights,
|
||||
|
||||
@@ -237,7 +237,7 @@ class VanillaPipeline(Pipeline):
|
||||
num_train_data=len(self.datamanager.train_dataset),
|
||||
metadata=self.datamanager.train_dataset.metadata,
|
||||
device=device,
|
||||
grad_scaler=grad_scaler
|
||||
grad_scaler=grad_scaler,
|
||||
)
|
||||
self.model.to(device)
|
||||
|
||||
|
||||
@@ -604,14 +604,6 @@ def colmap_to_json(
|
||||
cameras = read_cameras_binary(cameras_path)
|
||||
images = read_images_binary(images_path)
|
||||
|
||||
# Images were renamed to frame_{i:05d}.{ext} and
|
||||
# the filenames needs to be replaced in the transforms.json as well
|
||||
original_filenames = [x.name for x in images.values()]
|
||||
# Sort was used in nerfstudio.process_data.process_data_utils:get_image_filenames
|
||||
original_filenames.sort()
|
||||
# Build the map to the new filenames
|
||||
filename_map = {name: f"frame_{i+1:05d}{os.path.splitext(name)[-1]}" for i, name in enumerate(original_filenames)}
|
||||
|
||||
# Only supports one camera
|
||||
camera_params = cameras[1].params
|
||||
|
||||
@@ -627,8 +619,7 @@ def colmap_to_json(
|
||||
c2w = c2w[np.array([1, 0, 2, 3]), :]
|
||||
c2w[2, :] *= -1
|
||||
|
||||
name = filename_map[im_data.name]
|
||||
name = Path(f"./images/{name}")
|
||||
name = Path(f"./images/{im_data.name}")
|
||||
|
||||
frame = {
|
||||
"file_path": name.as_posix(),
|
||||
|
||||
@@ -19,6 +19,7 @@ from typing import Tuple
|
||||
|
||||
import torch
|
||||
from torchtyping import TensorType
|
||||
from typing_extensions import Literal
|
||||
|
||||
_USE_NERFACC = True
|
||||
|
||||
@@ -285,3 +286,28 @@ def safe_normalize(
|
||||
Normalized vectors.
|
||||
"""
|
||||
return vectors / (torch.norm(vectors, dim=-1, keepdim=True) + eps)
|
||||
|
||||
|
||||
def masked_reduction(
|
||||
input_tensor: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"], reduction_type: Literal["image", "batch"]
|
||||
):
|
||||
"""
|
||||
Whether to consolidate the input_tensor across the batch or across the image
|
||||
Args:
|
||||
input_tensor: input tensor
|
||||
mask: mask tensor
|
||||
reduction_type: either "batch" or "image"
|
||||
Returns:
|
||||
input_tensor: reduced input_tensor
|
||||
"""
|
||||
if reduction_type == "batch":
|
||||
# avoid division by 0 (if sum(M) = sum(sum(mask)) = 0: sum(image_loss) = 0)
|
||||
divisor = torch.sum(mask)
|
||||
input_tensor = 0 if divisor == 0 else torch.sum(input_tensor) / divisor
|
||||
elif reduction_type == "image":
|
||||
# avoid division by 0 (if M = sum(mask) = 0: image_loss = 0)
|
||||
valid = mask.nonzero()
|
||||
|
||||
input_tensor[valid] = input_tensor[valid] / mask[valid]
|
||||
input_tensor = torch.mean(input_tensor)
|
||||
return input_tensor
|
||||
@@ -12,6 +12,8 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# pylint: disable=too-many-lines
|
||||
|
||||
"""Code to interface with the `vis/` (the JS viewer).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
@@ -494,7 +496,7 @@ class ViewerState:
|
||||
|
||||
prompt = getattr(graph, "prompt", None)
|
||||
is_generative = prompt is not None
|
||||
|
||||
|
||||
if is_generative:
|
||||
self.vis["model/is_generative"].write(str(is_generative).lower())
|
||||
|
||||
|
||||
+2
-2
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "nerfstudio"
|
||||
version = "0.1.17"
|
||||
version = "0.1.18"
|
||||
description = "All-in-one repository for state-of-the-art NeRFs"
|
||||
readme = "README.md"
|
||||
license = { text="Apache 2.0"}
|
||||
@@ -100,7 +100,7 @@ ns-install-cli = "scripts.completions.install:entrypoint"
|
||||
ns-process-data = "scripts.process_data:entrypoint"
|
||||
ns-download-data = "scripts.downloads.download_data:entrypoint"
|
||||
ns-train = "scripts.train:entrypoint"
|
||||
ns-trace-sd = "scripts.trace_stablediff:entrypoint"
|
||||
ns-trace-sd = "scripts.generative.trace_stable_diffusion:entrypoint"
|
||||
ns-viewer = "scripts.viewer.run_viewer:entrypoint"
|
||||
ns-eval = "scripts.eval:entrypoint"
|
||||
ns-render = "scripts.render:entrypoint"
|
||||
|
||||
Executable → Regular
File mode changed.
Executable → Regular
+1
-1
@@ -28,7 +28,7 @@ fi
|
||||
method_opts=()
|
||||
if [ "$method_name" = "nerfacto" ]; then
|
||||
# https://github.com/nerfstudio-project/nerfstudio/issues/806#issuecomment-1284327844
|
||||
method_opts=(--pipeline.model.near-plane 2. --pipeline.model.far-plane 6. --pipeline.datamanager.camera-optimizer.mode off --pipeline.model.use-average-appearance-embedding False)
|
||||
method_opts=(--pipeline.model.background-color white --pipeline.model.near-plane 2. --pipeline.model.far-plane 6. --pipeline.datamanager.camera-optimizer.mode off --pipeline.model.use-average-appearance-embedding False)
|
||||
fi
|
||||
|
||||
shift $((OPTIND-1))
|
||||
|
||||
Executable → Regular
+199
-84
@@ -10,27 +10,29 @@ import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import List, Union
|
||||
from typing import List, Optional, Union
|
||||
|
||||
import tyro
|
||||
from rich.console import Console
|
||||
from rich.prompt import Confirm
|
||||
from typing_extensions import Literal, assert_never
|
||||
from typing_extensions import get_args as typing_get_args
|
||||
|
||||
if sys.version_info < (3, 10):
|
||||
import importlib_metadata
|
||||
else:
|
||||
from importlib import metadata as importlib_metadata
|
||||
|
||||
ConfigureMode = Literal["install", "uninstall"]
|
||||
ShellType = Literal["zsh", "bash"]
|
||||
|
||||
CONSOLE = Console(width=120)
|
||||
HEADER_LINE = "# Source nerfstudio autocompletions."
|
||||
|
||||
ENTRYPOINTS = [
|
||||
"ns-install-cli",
|
||||
"ns-process-data",
|
||||
"ns-download-data",
|
||||
"ns-train",
|
||||
"ns-eval",
|
||||
"ns-render",
|
||||
"ns-dev-test",
|
||||
]
|
||||
|
||||
def _get_all_entry_points() -> List[str]:
|
||||
entry_points = importlib_metadata.distribution("nerfstudio").entry_points
|
||||
return [x.name for x in entry_points]
|
||||
|
||||
|
||||
def _check_tyro_cli(script_path: pathlib.Path) -> bool:
|
||||
@@ -127,6 +129,62 @@ def _exclamation() -> str:
|
||||
return random.choice(["Cool", "Nice", "Neat", "Great", "Exciting", "Excellent", "Ok"]) + "!"
|
||||
|
||||
|
||||
def _get_deactivate_script(commands: List[str], shell: Optional[ShellType], add_header=True) -> str:
|
||||
if shell is None:
|
||||
# Install the universal script
|
||||
result_script = []
|
||||
for shell_type in typing_get_args(ShellType):
|
||||
result_script.append(f'if [ -n "${shell_type.upper()}_VERSION" ]; then')
|
||||
result_script.append(_get_deactivate_script(commands, shell_type, add_header=False))
|
||||
result_script.append("fi")
|
||||
source_lines = "\n".join(result_script)
|
||||
|
||||
elif shell == "zsh":
|
||||
source_lines = "\n".join([f"unset '_comps[{command}]' &> /dev/null" for command in commands])
|
||||
elif shell == "bash":
|
||||
source_lines = "\n".join([f"complete -r {command} &> /dev/null" for command in commands])
|
||||
else:
|
||||
assert_never(shell)
|
||||
|
||||
if add_header:
|
||||
source_lines = f"\n{HEADER_LINE}\n{source_lines}"
|
||||
return source_lines
|
||||
|
||||
|
||||
def _get_source_script(completions_dir: pathlib.Path, shell: Optional[ShellType], add_header=True) -> str:
|
||||
if shell is None:
|
||||
# Install the universal script
|
||||
result_script = []
|
||||
for shell_type in typing_get_args(ShellType):
|
||||
result_script.append(f'if [ -n "${shell_type.upper()}_VERSION" ]; then')
|
||||
result_script.append(_get_source_script(completions_dir, shell_type, add_header=False))
|
||||
result_script.append("fi")
|
||||
source_lines = "\n".join(result_script)
|
||||
|
||||
elif shell == "zsh":
|
||||
source_lines = "\n".join(
|
||||
[
|
||||
"if ! command -v compdef &> /dev/null; then",
|
||||
" autoload -Uz compinit",
|
||||
" compinit",
|
||||
"fi",
|
||||
f"source {completions_dir / 'setup.zsh'}",
|
||||
]
|
||||
)
|
||||
elif shell == "bash":
|
||||
source_lines = "\n".join(
|
||||
[
|
||||
f"source {completions_dir / 'setup.bash'}",
|
||||
]
|
||||
)
|
||||
else:
|
||||
assert_never(shell)
|
||||
|
||||
if add_header:
|
||||
source_lines = f"\n{HEADER_LINE}\n{source_lines}"
|
||||
return source_lines
|
||||
|
||||
|
||||
def _update_rc(
|
||||
completions_dir: pathlib.Path,
|
||||
mode: ConfigureMode,
|
||||
@@ -141,36 +199,13 @@ def _update_rc(
|
||||
"""
|
||||
|
||||
# Install or uninstall `source_line`.
|
||||
header_line = "# Source nerfstudio autocompletions."
|
||||
if shell == "zsh":
|
||||
source_lines = "\n".join(
|
||||
[
|
||||
"",
|
||||
header_line,
|
||||
"if ! command -v compdef &> /dev/null; then",
|
||||
" autoload -Uz compinit",
|
||||
" compinit",
|
||||
"fi",
|
||||
f"source {completions_dir / 'setup.zsh'}",
|
||||
]
|
||||
)
|
||||
elif shell == "bash":
|
||||
source_lines = "\n".join(
|
||||
[
|
||||
"",
|
||||
header_line,
|
||||
f"source {completions_dir / 'setup.bash'}",
|
||||
]
|
||||
)
|
||||
else:
|
||||
assert_never(shell)
|
||||
|
||||
source_lines = _get_source_script(completions_dir, shell)
|
||||
rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
|
||||
|
||||
# Always try to uninstall previous completions.
|
||||
rc_source = rc_path.read_text()
|
||||
while header_line in rc_source:
|
||||
before_install, _, after_install = rc_source.partition(header_line)
|
||||
while HEADER_LINE in rc_source:
|
||||
before_install, _, after_install = rc_source.partition(HEADER_LINE)
|
||||
source_file, _, after_install = after_install.partition("\nsource ")[2].partition("\n")
|
||||
assert source_file.endswith(f"/completions/setup.{shell}")
|
||||
rc_source = before_install + after_install
|
||||
@@ -189,6 +224,108 @@ def _update_rc(
|
||||
assert mode == "uninstall"
|
||||
|
||||
|
||||
def _update_conda_scripts(
|
||||
commands: List[str],
|
||||
completions_dir: pathlib.Path,
|
||||
mode: ConfigureMode,
|
||||
) -> None:
|
||||
"""Try to add a `source /.../completions/setup.{shell}` line automatically to conda's activation scripts.
|
||||
|
||||
Args:
|
||||
completions_dir: Path to location of this script.
|
||||
mode: Install or uninstall completions.
|
||||
"""
|
||||
|
||||
# Install or uninstall `source_line`.
|
||||
activate_source_lines = _get_source_script(completions_dir, None)
|
||||
deactivate_source_lines = _get_deactivate_script(commands, None)
|
||||
|
||||
conda_path = pathlib.Path(os.environ["CONDA_PREFIX"])
|
||||
activate_path = conda_path / "etc/conda/activate.d/nerfstudio_activate.sh"
|
||||
deactivate_path = conda_path / "etc/conda/deactivate.d/nerfstudio_deactivate.sh"
|
||||
if mode == "uninstall":
|
||||
if activate_path.exists():
|
||||
os.remove(activate_path)
|
||||
if deactivate_path.exists():
|
||||
os.remove(deactivate_path)
|
||||
CONSOLE.log(f":broom: Existing completions uninstalled from {conda_path}.")
|
||||
elif mode == "install":
|
||||
# Install completions.
|
||||
activate_path.parent.mkdir(exist_ok=True, parents=True)
|
||||
deactivate_path.parent.mkdir(exist_ok=True, parents=True)
|
||||
with activate_path.open("w+", encoding="utf8") as f:
|
||||
f.write(activate_source_lines)
|
||||
with deactivate_path.open("w+", encoding="utf8") as f:
|
||||
f.write(deactivate_source_lines)
|
||||
CONSOLE.log(
|
||||
f":person_gesturing_ok: Completions installed to {conda_path}. {_exclamation()} Reactivate the environment"
|
||||
" to try them out."
|
||||
)
|
||||
else:
|
||||
assert_never(mode)
|
||||
|
||||
|
||||
def _get_conda_path() -> Optional[pathlib.Path]:
|
||||
"""
|
||||
Returns the path to the conda environment if
|
||||
the nerfstudio package is installed in one.
|
||||
"""
|
||||
conda_path = None
|
||||
if "CONDA_PREFIX" in os.environ:
|
||||
# Conda is active, we will check if the Nerfstudio is installed in the conda env.
|
||||
distribution = importlib_metadata.distribution("nerfstudio")
|
||||
if str(distribution.locate_file("nerfstudio")).startswith(os.environ["CONDA_PREFIX"]):
|
||||
conda_path = pathlib.Path(os.environ["CONDA_PREFIX"])
|
||||
return conda_path
|
||||
|
||||
|
||||
def _generate_completions_files(
|
||||
completions_dir: pathlib.Path,
|
||||
scripts_dir: pathlib.Path,
|
||||
shells_supported: List[ShellType],
|
||||
shells_found: List[ShellType],
|
||||
) -> None:
|
||||
# Set to True to install completions for scripts as well.
|
||||
include_scripts = False
|
||||
|
||||
# Find tyro CLIs.
|
||||
script_paths = list(filter(_check_tyro_cli, scripts_dir.glob("**/*.py"))) if include_scripts else []
|
||||
script_names = tuple(p.name for p in script_paths)
|
||||
assert len(set(script_names)) == len(script_names)
|
||||
|
||||
# Get existing completion files.
|
||||
existing_completions = set()
|
||||
for shell in shells_supported:
|
||||
target_dir = completions_dir / shell
|
||||
if target_dir.exists():
|
||||
existing_completions |= set(target_dir.glob("*"))
|
||||
|
||||
# Get all entry_points.
|
||||
entry_points = _get_all_entry_points()
|
||||
|
||||
# Run generation jobs.
|
||||
concurrent_executor = concurrent.futures.ThreadPoolExecutor()
|
||||
with CONSOLE.status("[bold]:writing_hand: Generating completions...", spinner="bouncingBall"):
|
||||
completion_paths = list(
|
||||
concurrent_executor.map(
|
||||
lambda path_or_entrypoint_and_shell: _generate_completion(
|
||||
path_or_entrypoint_and_shell[0], path_or_entrypoint_and_shell[1], completions_dir
|
||||
),
|
||||
itertools.product(script_paths + entry_points, shells_found),
|
||||
)
|
||||
)
|
||||
|
||||
# Delete obsolete completion files.
|
||||
for unexpected_path in set(p.absolute() for p in existing_completions) - set(
|
||||
p.absolute() for p in completion_paths
|
||||
):
|
||||
if unexpected_path.is_dir():
|
||||
shutil.rmtree(unexpected_path)
|
||||
elif unexpected_path.exists():
|
||||
unexpected_path.unlink()
|
||||
CONSOLE.log(f":broom: Deleted {unexpected_path}.")
|
||||
|
||||
|
||||
def main(mode: ConfigureMode = "install") -> None:
|
||||
"""Main script.
|
||||
|
||||
@@ -201,16 +338,24 @@ def main(mode: ConfigureMode = "install") -> None:
|
||||
CONSOLE.log("[bold red]$HOME is not set. Exiting.")
|
||||
return
|
||||
|
||||
# Get conda path if in conda environment.
|
||||
conda_path = _get_conda_path()
|
||||
|
||||
# Try to locate the user's bashrc or zshrc.
|
||||
shells_supported: List[ShellType] = ["zsh", "bash"]
|
||||
shells_found: List[ShellType] = []
|
||||
for shell in shells_supported:
|
||||
rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
|
||||
if not rc_path.exists():
|
||||
CONSOLE.log(f":person_shrugging: {rc_path.name} not found, skipping.")
|
||||
else:
|
||||
CONSOLE.log(f":mag: Found {rc_path.name}!")
|
||||
shells_found.append(shell)
|
||||
shells_supported: List[ShellType] = list(typing_get_args(ShellType))
|
||||
if conda_path is not None:
|
||||
# Running in conda; we have to support all shells.
|
||||
shells_found = shells_supported
|
||||
CONSOLE.log(f":mag: Detected conda environment {conda_path}!")
|
||||
else:
|
||||
shells_found: List[ShellType] = []
|
||||
for shell in shells_supported:
|
||||
rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
|
||||
if not rc_path.exists():
|
||||
CONSOLE.log(f":person_shrugging: {rc_path.name} not found, skipping.")
|
||||
else:
|
||||
CONSOLE.log(f":mag: Found {rc_path.name}!")
|
||||
shells_found.append(shell)
|
||||
|
||||
# Get scripts/ directory.
|
||||
completions_dir = pathlib.Path(__file__).absolute().parent
|
||||
@@ -230,48 +375,18 @@ def main(mode: ConfigureMode = "install") -> None:
|
||||
else:
|
||||
CONSOLE.log(f":heavy_check_mark: No existing completions at: {target_dir}.")
|
||||
elif mode == "install":
|
||||
# Set to True to install completions for scripts as well.
|
||||
include_scripts = False
|
||||
|
||||
# Find tyro CLIs.
|
||||
script_paths = list(filter(_check_tyro_cli, scripts_dir.glob("**/*.py"))) if include_scripts else []
|
||||
script_names = tuple(p.name for p in script_paths)
|
||||
assert len(set(script_names)) == len(script_names)
|
||||
|
||||
# Get existing completion files.
|
||||
existing_completions = set()
|
||||
for shell in shells_supported:
|
||||
target_dir = completions_dir / shell
|
||||
if target_dir.exists():
|
||||
existing_completions |= set(target_dir.glob("*"))
|
||||
|
||||
# Run generation jobs.
|
||||
concurrent_executor = concurrent.futures.ThreadPoolExecutor()
|
||||
with CONSOLE.status("[bold]:writing_hand: Generating completions...", spinner="bouncingBall"):
|
||||
completion_paths = list(
|
||||
concurrent_executor.map(
|
||||
lambda path_or_entrypoint_and_shell: _generate_completion(
|
||||
path_or_entrypoint_and_shell[0], path_or_entrypoint_and_shell[1], completions_dir
|
||||
),
|
||||
itertools.product(script_paths + ENTRYPOINTS, shells_found),
|
||||
)
|
||||
)
|
||||
|
||||
# Delete obsolete completion files.
|
||||
for unexpected_path in set(p.absolute() for p in existing_completions) - set(
|
||||
p.absolute() for p in completion_paths
|
||||
):
|
||||
if unexpected_path.is_dir():
|
||||
shutil.rmtree(unexpected_path)
|
||||
elif unexpected_path.exists():
|
||||
unexpected_path.unlink()
|
||||
CONSOLE.log(f":broom: Deleted {unexpected_path}.")
|
||||
_generate_completions_files(completions_dir, scripts_dir, shells_supported, shells_found)
|
||||
else:
|
||||
assert_never(mode)
|
||||
|
||||
# Install or uninstall from bashrc/zshrc.
|
||||
for shell in shells_found:
|
||||
_update_rc(completions_dir, mode, shell)
|
||||
if conda_path is not None:
|
||||
# In conda environment we add the completitions activation scripts.
|
||||
commands = _get_all_entry_points()
|
||||
_update_conda_scripts(commands, completions_dir, mode)
|
||||
else:
|
||||
# Install or uninstall from bashrc/zshrc.
|
||||
for shell in shells_found:
|
||||
_update_rc(completions_dir, mode, shell)
|
||||
|
||||
CONSOLE.print("[bold]All done![/bold]")
|
||||
|
||||
|
||||
Executable → Regular
File mode changed.
Executable → Regular
File mode changed.
Executable → Regular
File mode changed.
Executable → Regular
File mode changed.
@@ -2,6 +2,7 @@
|
||||
|
||||
import functools
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import appdirs
|
||||
@@ -9,6 +10,8 @@ import torch
|
||||
import tyro
|
||||
from rich.console import Console
|
||||
|
||||
from nerfstudio.generative.stable_diffusion import SD_IDENTIFIERS
|
||||
|
||||
CONSOLE = Console(width=120)
|
||||
|
||||
try:
|
||||
@@ -20,55 +23,77 @@ except ImportError:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def jit_unet(save_dir: Path = Path(appdirs.user_data_dir("nerfstudio"))):
|
||||
"""Trace Stable Diffusion UNet for speed improvement
|
||||
@dataclass
|
||||
class TraceSD:
|
||||
"""Traces the Stable Diffusion U-Net for better performance."""
|
||||
|
||||
Args:
|
||||
save_dir: directory to save the traced model
|
||||
"""
|
||||
sd_version: str
|
||||
"""Stable diffusion version."""
|
||||
save_dir: Path = Path(appdirs.user_data_dir("nerfstudio"))
|
||||
"""Directory to save the traced model."""
|
||||
|
||||
# torch disable grad
|
||||
torch.set_grad_enabled(False)
|
||||
def main(self):
|
||||
"""Trace the Stable Diffusion U-Net and save it to disk."""
|
||||
# torch disable grad
|
||||
torch.set_grad_enabled(False)
|
||||
|
||||
# load inputs
|
||||
def generate_inputs():
|
||||
sample = torch.randn(2, 4, 64, 64).half().cuda()
|
||||
timestep = torch.rand(1).half().cuda() * 999
|
||||
encoder_hidden_states = torch.randn(2, 77, 768).half().cuda()
|
||||
return sample, timestep, encoder_hidden_states
|
||||
if self.sd_version not in SD_IDENTIFIERS:
|
||||
CONSOLE.print(f"[bold red]Invalid Stable Diffusion version; choose from {set(SD_IDENTIFIERS.keys())}")
|
||||
sys.exit(1)
|
||||
|
||||
pipe = StableDiffusionPipeline.from_pretrained(
|
||||
"runwayml/stable-diffusion-v1-5",
|
||||
torch_dtype=torch.float16,
|
||||
).to("cuda")
|
||||
sd_id = SD_IDENTIFIERS[self.sd_version]
|
||||
pipe = StableDiffusionPipeline.from_pretrained(
|
||||
sd_id,
|
||||
torch_dtype=torch.float16,
|
||||
)
|
||||
assert pipe is not None
|
||||
pipe = pipe.to("cuda")
|
||||
pipe.enable_attention_slicing()
|
||||
|
||||
pipe.enable_attention_slicing()
|
||||
# load inputs
|
||||
def generate_inputs():
|
||||
sample = torch.randn(2, 4, 64, 64).half().cuda()
|
||||
timestep = torch.rand(1).half().cuda() * 999
|
||||
|
||||
unet = pipe.unet
|
||||
unet.eval()
|
||||
unet.to(memory_format=torch.channels_last) # use channels_last memory format
|
||||
unet.forward = functools.partial(unet.forward, return_dict=False) # set return_dict=False as default
|
||||
# get shape of encoder hidden state (varies by SD version)
|
||||
text_input = pipe.tokenizer("", return_tensors="pt")
|
||||
_, w, d = pipe.text_encoder(text_input.input_ids.cuda())[0].shape
|
||||
|
||||
# warmup
|
||||
inputs = None
|
||||
for _ in range(3):
|
||||
with torch.inference_mode():
|
||||
inputs = generate_inputs()
|
||||
_ = unet(*inputs)
|
||||
encoder_hidden_states = torch.randn(2, w, d).half().cuda()
|
||||
return sample, timestep, encoder_hidden_states
|
||||
|
||||
# trace
|
||||
print("tracing..")
|
||||
assert inputs is not None
|
||||
unet_traced = torch.jit.trace(unet, inputs)
|
||||
unet_traced.eval()
|
||||
print("done tracing")
|
||||
unet = pipe.unet
|
||||
unet.eval()
|
||||
unet.to(memory_format=torch.channels_last) # use channels_last memory format
|
||||
unet.forward = functools.partial(unet.forward, return_dict=False) # set return_dict=False as default
|
||||
|
||||
# save the traced model
|
||||
unet_traced_filename = save_dir / "sd_unet_traced.pt"
|
||||
if not unet_traced_filename.exists():
|
||||
unet_traced_filename.parent.mkdir(parents=True, exist_ok=True)
|
||||
unet_traced.save(unet_traced_filename)
|
||||
# warmup
|
||||
inputs = None
|
||||
for _ in range(3):
|
||||
with torch.inference_mode():
|
||||
inputs = generate_inputs()
|
||||
_ = unet(*inputs)
|
||||
|
||||
# trace
|
||||
print("tracing..")
|
||||
assert inputs is not None
|
||||
unet_traced = torch.jit.trace(unet, inputs)
|
||||
unet_traced.eval() # type: ignore
|
||||
print("done tracing")
|
||||
|
||||
# save the traced model
|
||||
filename_sd_id = sd_id.split("/")[-1]
|
||||
unet_traced_filename = self.save_dir / f"{filename_sd_id}_unet_traced.pt"
|
||||
if not unet_traced_filename.exists():
|
||||
unet_traced_filename.parent.mkdir(parents=True, exist_ok=True)
|
||||
unet_traced.save(unet_traced_filename) # type: ignore
|
||||
|
||||
|
||||
def entrypoint():
|
||||
"""Entrypoint for use with pyproject scripts."""
|
||||
tyro.extras.set_accent_color("bright_yellow")
|
||||
tyro.cli(TraceSD).main()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
tyro.cli(jit_unet)
|
||||
entrypoint()
|
||||
Executable → Regular
File mode changed.
Executable → Regular
File mode changed.
@@ -1,82 +0,0 @@
|
||||
"""Trace Stable Diffusion UNet for speed improvement"""
|
||||
|
||||
import functools
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import appdirs
|
||||
import torch
|
||||
import tyro
|
||||
from diffusers import StableDiffusionPipeline
|
||||
from rich.console import Console
|
||||
|
||||
from nerfstudio.generative.stable_diffusion import SD_IDENTIFIERS
|
||||
|
||||
CONSOLE = Console(width=120)
|
||||
|
||||
@dataclass
|
||||
class TraceSD:
|
||||
"""Traces the Stable Diffusion U-Net for better performance."""
|
||||
|
||||
sd_version: str
|
||||
"""Stable diffusion version."""
|
||||
|
||||
def main(self):
|
||||
# torch disable grad
|
||||
torch.set_grad_enabled(False)
|
||||
|
||||
if self.sd_version not in SD_IDENTIFIERS.keys():
|
||||
CONSOLE.print(f"[bold red]Invalid Stable Diffusion version; choose from {set(SD_IDENTIFIERS.keys())}")
|
||||
sys.exit(1)
|
||||
|
||||
sd_id = SD_IDENTIFIERS[self.sd_version]
|
||||
pipe = StableDiffusionPipeline.from_pretrained(
|
||||
sd_id,
|
||||
torch_dtype=torch.float16,
|
||||
).to("cuda")
|
||||
pipe.enable_attention_slicing()
|
||||
|
||||
# load inputs
|
||||
def generate_inputs():
|
||||
sample = torch.randn(2, 4, 64, 64).half().cuda()
|
||||
timestep = torch.rand(1).half().cuda() * 999
|
||||
|
||||
# get shape of encoder hidden state (varies by SD version)
|
||||
text_input = pipe.tokenizer("", return_tensors="pt")
|
||||
_, w, d = pipe.text_encoder(text_input.input_ids.cuda())[0].shape
|
||||
|
||||
encoder_hidden_states = torch.randn(2, w, d).half().cuda()
|
||||
return sample, timestep, encoder_hidden_states
|
||||
|
||||
unet = pipe.unet
|
||||
unet.eval()
|
||||
unet.to(memory_format=torch.channels_last) # use channels_last memory format
|
||||
unet.forward = functools.partial(unet.forward, return_dict=False) # set return_dict=False as default
|
||||
|
||||
# warmup
|
||||
for _ in range(3):
|
||||
with torch.inference_mode():
|
||||
inputs = generate_inputs()
|
||||
_ = unet(*inputs)
|
||||
|
||||
# trace
|
||||
print("tracing..")
|
||||
unet_traced = torch.jit.trace(unet, inputs)
|
||||
unet_traced.eval()
|
||||
print("done tracing")
|
||||
|
||||
# save the traced model
|
||||
filename_sd_id = sd_id.split('/')[-1]
|
||||
unet_traced_filename = Path(appdirs.user_data_dir("nerfstudio")) / f'{filename_sd_id}_unet_traced.pt'
|
||||
if not unet_traced_filename.exists():
|
||||
unet_traced_filename.parent.mkdir(parents=True, exist_ok=True)
|
||||
unet_traced.save(unet_traced_filename)
|
||||
|
||||
def entrypoint():
|
||||
"""Entrypoint for use with pyproject scripts."""
|
||||
tyro.extras.set_accent_color("bright_yellow")
|
||||
tyro.cli(TraceSD).main()
|
||||
|
||||
if __name__ == "__main__":
|
||||
entrypoint()
|
||||
Executable → Regular
File mode changed.
Executable → Regular
File mode changed.
Reference in new issue
Block a user