Merge remote-tracking branch 'origin' into generative

This commit is contained in:
Matt Tancik committed 2023-02-20 09:48:41 -08:00
commit 36be1fe4b0
32 files changed
+623 -140

No files matched your search

+13
View File
@@ -0,0 +1,13 @@
{
"name": "NerfStudio Dev Container",
"dockerComposeFile": "docker-compose.yml",
"service": "nerfstudio",
"workspaceFolder": "/workspace",
"shutdownAction": "stopCompose",
"extensions": [
"mutantdino.resourcemonitor",
"ms-azuretools.vscode-docker",
"nvidia.nsight-vscode-edition",
"ms-python.python"
]
}
+28
View File
@@ -0,0 +1,28 @@
version: "3.8"
services:
nerfstudio:
shm_size: '12gb'
image: dromni/nerfstudio:0.1.17
# build:
# context: ..
# dockerfile: ./Dockerfile
stdin_open: true
tty: true
environment:
NVIDIA_DRIVER_CAPABILITIES: compute,utility,graphics
DISPLAY: $DISPLAY
ports:
- "7007:7007"
volumes:
- ../:/workspace/
- /tmp/.X11-unix:/tmp/.X11-unix
working_dir: /workspace/
command: /bin/bash
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: 1
capabilities: [ gpu ]
+27 -13
View File
@@ -1,14 +1,16 @@
# Define base image.
FROM nvidia/cuda:11.7.1-devel-ubuntu22.04
FROM nvidia/cuda:11.8.0-devel-ubuntu22.04
# Variables used at build time.
## CUDA architectures, required by Colmap and tiny-cuda-nn.
## NOTE: All commonly used GPU architectures are included and supported here. To speedup the image build process remove all architectures but the one of your explicit GPU. Find details here: https://developer.nvidia.com/cuda-gpus (8.6 translates to 86 in the line below) or in the docs.
ARG CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37
# Set environment variables.
## Set non-interactive to prevent asking for user inputs blocking image creation.
ENV DEBIAN_FRONTEND=noninteractive
## Set timezone as it is required by some packages.
ENV TZ=Europe/Berlin
## CUDA architectures, required by tiny-cuda-nn.
## NOTE: All commonly used GPU architectures are included and supported here. To speedup the image build process remove all architectures but the one of your explicit GPU. Find details here: https://developer.nvidia.com/cuda-gpus (8.6 translates to 86 in the line below) or in the docs.
ENV TCNN_CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37
## CUDA Home, required to find CUDA in some packages.
ENV CUDA_HOME="/usr/local/cuda"
@@ -17,6 +19,7 @@ RUN apt-get update && \
apt-get install -y --no-install-recommends \
build-essential \
cmake \
curl \
ffmpeg \
git \
libatlas-base-dev \
@@ -27,6 +30,7 @@ RUN apt-get update && \
libboost-test-dev \
libcgal-dev \
libeigen3-dev \
libflann-dev \
libfreeimage-dev \
libgflags-dev \
libglew-dev \
@@ -34,15 +38,18 @@ RUN apt-get update && \
libmetis-dev \
libprotobuf-dev \
libqt5opengl5-dev \
libsqlite3-dev \
libsuitesparse-dev \
nano \
protobuf-compiler \
python3.10-dev \
python3-pip \
qtbase5-dev \
sudo \
wget && \
rm -rf /var/lib/apt/lists/*
# Install GLOG (required by ceres).
RUN git clone --branch v0.6.0 https://github.com/google/glog.git --single-branch && \
cd glog && \
@@ -69,22 +76,28 @@ RUN git clone --branch 2.1.0 https://ceres-solver.googlesource.com/ceres-solver.
rm -rf ceres-solver
# Install colmap.
RUN git clone --branch 3.7 https://github.com/colmap/colmap.git --single-branch && \
RUN git clone --branch 3.8 https://github.com/colmap/colmap.git --single-branch && \
cd colmap && \
mkdir build && \
cd build && \
cmake .. -DCUDA_ENABLED=ON \
-DCUDA_NVCC_FLAGS="--std c++14" && \
-DCUDA_NVCC_FLAGS="--std c++14" \
-DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES} && \
make -j && \
make install && \
cd ../.. && \
rm -rf colmap
# Create non root user and setup environment.
RUN useradd -m -d /home/user -u 1000 user
RUN useradd -m -d /home/user -g root -G sudo -u 1000 user
RUN usermod -aG sudo user
# Set user password
RUN echo "user:user" | chpasswd
# Ensure sudo group users are not asked for a password when using sudo command by ammending sudoers file
RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
# Switch to new uer and workdir.
USER 1000:1000
USER 1000
WORKDIR /home/user
# Add local user binary folder to PATH variable.
@@ -93,16 +106,17 @@ SHELL ["/bin/bash", "-c"]
# Upgrade pip and install packages.
RUN python3.10 -m pip install --upgrade pip setuptools pathtools promise
# Install pytorch and submodules.
RUN python3.10 -m pip install torch==1.12.1+cu116 torchvision==0.13.1+cu116 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu116
# Install tynyCUDNN.
# Install pytorch and submodules (Currently, we still use cu116 which is the latest version for toch 1.12.1 and is compatible with CUDA 11.8).
RUN python3.10 -m pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116
# Install tynyCUDNN (we need to set the target architectures as environment variable first).
ENV TCNN_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES}
RUN python3.10 -m pip install git+https://github.com/NVlabs/tiny-cuda-nn.git#subdirectory=bindings/torch
# Copy nerfstudio folder and give ownership to user.
ADD . /home/user/nerfstudio
USER root
RUN chown -R user:user /home/user/nerfstudio
USER 1000:1000
RUN chown -R user /home/user/nerfstudio
USER 1000
# Install nerfstudio dependencies.
RUN cd nerfstudio && \
+2
View File
@@ -302,6 +302,8 @@ We provide the following support structures to make life easier for getting star
# Citation
You can find a paper writeup of the framework on [arXiv](https://arxiv.org/abs/2302.04264).
If you use this library or find the documentation useful for your research, please consider citing:
```
+2
View File
@@ -155,6 +155,8 @@ We'll be constantly growing this list! So make sure to check back in to see our
## Citation
You can find a paper writeup of the framework on [arXiv](https://arxiv.org/abs/2302.04264).
If you use this library or find the documentation useful for your research, please consider citing:
```none
+8 -6
View File
@@ -92,8 +92,8 @@ pip install -e .[docs]
## Use docker image
Instead of installing and compiling prerequisites, setting up the environment and installing dependencies, a ready to use docker image is provided.
### Prerequisites
Docker ([get docker](https://docs.docker.com/get-docker/)) and nvidia GPU drivers ([get nvidia drivers](https://www.nvidia.de/Download/index.aspx?lang=de)), capable of working with CUDA 11.7, must be installed.
The docker image can then either be pulled from [here](https://hub.docker.com/r/dromni/nerfstudio/tags) (replace <version> with the actual version, e.g. 0.1.10)
Docker ([get docker](https://docs.docker.com/get-docker/)) and nvidia GPU drivers ([get nvidia drivers](https://www.nvidia.de/Download/index.aspx?lang=de)), capable of working with CUDA 11.8, must be installed.
The docker image can then either be pulled from [here](https://hub.docker.com/r/dromni/nerfstudio/tags) (replace <version> with the actual version, e.g. 0.1.17)
```bash
docker pull dromni/nerfstudio:<version>
```
@@ -110,13 +110,15 @@ docker run --gpus all \ # Give the conta
-p 7007:7007 \ # Map port from local machine to docker container (required to access the web interface/UI).
--rm \ # Remove container after it is closed (recommended).
-it \ # Start container in interactive mode.
nerfstudio # Docker image name
dromni/nerfstudio:<tag> # Docker image name if you pulled from docker hub.
<--- OR --->
nerfstudio # Docker image tag if you built the image from the Dockerfile by yourself using the command from above.
```
### Call nerfstudio commands directly
Besides, the container can also directly be used by adding the nerfstudio command to the end.
```bash
docker run --gpus all -v /folder/of/your/data:/workspace/ -v /home/<YOUR_USER>/.cache/:/home/user/.cache/ -p 7007:7007 --rm -it # Parameters.
nerfstudio \ # Docker image name
dromni/nerfstudio:<tag> \ # Docker image name
ns-process-data video --data /workspace/video.mp4 # Smaple command of nerfstudio.
```
### Note
@@ -125,10 +127,10 @@ docker run --gpus all -v /folder/of/your/data:/workspace/ -v /home/<YOUR_USER>/.
- Always use full paths, relative paths are known to create issues when being used in mounts into docker.
- Everything inside the container, what is not in a mounted folder (workspace in the above example), will be permanently removed after destroying the container. Always do all your tasks and output folder in workdir!
- The user inside the container is called 'user' and is mapped to the local user with ID 1000 (usually the first non-root user on Linux systems).
- The container currently is based on nvidia/cuda:11.7.1-devel-ubuntu22.04, consequently it comes with CUDA 11.7 which must be supported by the nvidia driver. No local CUDA installation is required or will be affected by using the docker image.
- The container currently is based on nvidia/cuda:11.8.0-devel-ubuntu22.04, consequently it comes with CUDA 11.8 which must be supported by the nvidia driver. No local CUDA installation is required or will be affected by using the docker image.
- The docker image (respectively Ubuntu 22.04) comes with Python3.10, no older version of Python is installed.
- If you call the container with commands directly, you still might want to add the interactive terminal ('-it') flag to get live log outputs of the nerfstudio scripts. In case the container is used in an automated environment the flag should be discarded.
- The current version of docker is built for multi-architecture (CUDA architectures) use. The target architecture must be defined at build time for tinyCUDNN to be able to compile properly. If your GPU architecture is not covered by the following table you need to replace the number in the line ```ENV TCNN_CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37``` to you specific architecture. It also is a good idea to remove all architectures but yours (e.g. ```ENV TCNN_CUDA_ARCHITECTURES=86```) to speedup the docker build a lot.
- The current version of docker is built for multi-architecture (CUDA architectures) use. The target architecture(s) must be defined at build time for Colmap and tinyCUDNN to be able to compile properly. If your GPU architecture is not covered by the following table you need to replace the number in the line ```ARG CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37``` to your specific architecture. It also is a good idea to remove all architectures but yours (e.g. ```ARG CUDA_ARCHITECTURES=86```) to speedup the docker build process a lot.
**Currently supported CUDA architectures in the docker image**
+2
View File
@@ -12,6 +12,7 @@ Here are the popular commands that we offer. If you've cloned the repo, you can
| ns-process-data | Generate a dataset from your own data | scripts/process_data.py |
| ns-download-data | Download existing captures | scripts/downloads/download_data.py |
| ns-train | Generate a NeRF | scripts/train.py |
| ns-viewer | View a trained NeRF | scripts/viewer/run_viewer.py |
| ns-eval | Run evaluation metrics for your Model | scripts/eval.py |
| ns-render | Render out a video of your NeRF | scripts/render.py |
| ns-export | Export a NeRF into other formats | scripts/exporter.py |
@@ -24,6 +25,7 @@ ns_process_data
ns_download_data
ns_train
ns_render
ns_viewer
ns_export
ns_eval
```
+9
View File
@@ -0,0 +1,9 @@
# ns-viewer
```{eval-rst}
.. argparse::
:module: scripts.viewer.run_viewer
:func: get_parser_fn
:prog: ns-viewer
:nodefault:
```
@@ -51,7 +51,11 @@ from nerfstudio.data.dataparsers.phototourism_dataparser import (
)
from nerfstudio.data.dataparsers.sdfstudio_dataparser import SDFStudioDataParserConfig
from nerfstudio.data.datasets.base_dataset import InputDataset
from nerfstudio.data.pixel_samplers import EquirectangularPixelSampler, PixelSampler
from nerfstudio.data.pixel_samplers import (
EquirectangularPixelSampler,
PatchPixelSampler,
PixelSampler,
)
from nerfstudio.data.utils.dataloaders import (
CacheDataloader,
FixedIndicesEvalDataloader,
@@ -283,6 +287,8 @@ class VanillaDataManagerConfig(InstantiateConfig):
"""The scale factor for scaling spatial data such as images, mask, semantics
along with relevant information about camera intrinsics
"""
patch_size: int = 1
"""Size of patch to sample from. If >1, patch-based sampling will be used."""
class VanillaDataManager(DataManager): # pylint: disable=abstract-method
@@ -346,6 +352,9 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method
self, dataset: InputDataset, *args: Any, **kwargs: Any
) -> PixelSampler:
"""Infer pixel sampler to use."""
if self.config.patch_size > 1:
return PatchPixelSampler(*args, **kwargs, patch_size=self.config.patch_size)
# If all images are equirectangular, use equirectangular pixel sampler
is_equirectangular = dataset.cameras.camera_type == CameraType.EQUIRECTANGULAR.value
if is_equirectangular.all():
+51
View File
@@ -243,3 +243,54 @@ class EquirectangularPixelSampler(PixelSampler): # pylint: disable=too-few-publ
).long()
return indices
class PatchPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods
"""Samples 'pixel_batch's from 'image_batch's. Samples square patches
from the images randomly. Useful for patch-based losses.
Args:
num_rays_per_batch: number of rays to sample per batch
keep_full_image: whether or not to include a reference to the full image in returned batch
patch_size: side length of patch. This must be consistent in the method
config in order for samples to be reshaped into patches correctly.
"""
def __init__(self, num_rays_per_batch: int, keep_full_image: bool = False, **kwargs) -> None:
self.patch_size = kwargs["patch_size"]
num_rays = (num_rays_per_batch // (self.patch_size**2)) * (self.patch_size**2)
super().__init__(num_rays, keep_full_image, **kwargs)
# overrides base method
def sample_method( # pylint: disable=no-self-use
self,
batch_size: int,
num_images: int,
image_height: int,
image_width: int,
mask: Optional[TensorType] = None,
device: Union[torch.device, str] = "cpu",
) -> TensorType["batch_size", 3]:
if mask:
# Note: if there is a mask, sampling reduces back to uniform sampling
indices = super().sample_method(batch_size, num_images, image_height, image_width, mask=mask, device=device)
else:
sub_bs = batch_size // (self.patch_size**2)
indices = torch.rand((sub_bs, 3), device=device) * torch.tensor(
[num_images, image_height - self.patch_size, image_width - self.patch_size],
device=device,
)
indices = indices.view(sub_bs, 1, 1, 3).broadcast_to(sub_bs, self.patch_size, self.patch_size, 3).clone()
yys, xxs = torch.meshgrid(
torch.arange(self.patch_size, device=device), torch.arange(self.patch_size, device=device)
)
indices[:, ..., 1] += yys
indices[:, ..., 2] += xxs
indices = torch.floor(indices).long()
indices = indices.flatten(0, 2)
return indices
+4
View File
@@ -151,6 +151,10 @@ def generate_point_cloud(
CONSOLE.print(f"Please set --normal_output_name to one of: {outputs.keys()}", justify="center")
sys.exit(1)
normal = outputs[normal_output_name]
assert (
torch.min(normal) >= 0.0 and torch.max(normal) <= 1.0
), "Normal values from method output must be in [0, 1]"
normal = (normal * 2.0) - 1.0
point = ray_bundle.origins + ray_bundle.directions * depth
if use_bounding_box:
+217
View File
@@ -20,8 +20,10 @@ from enum import Enum
import torch
from torch import nn
from torchtyping import TensorType
from typing_extensions import Literal
from nerfstudio.cameras.rays import RaySamples
from nerfstudio.utils.math import masked_reduction
L1Loss = nn.L1Loss
MSELoss = nn.MSELoss
@@ -308,3 +310,218 @@ def depth_loss(
return urban_radiance_field_depth_loss(weights, termination_depth, predicted_depth, steps, sigma)
raise NotImplementedError("Provided depth loss type not implemented.")
def monosdf_normal_loss(
normal_pred: TensorType["num_samples", 3], normal_gt: TensorType["num_samples", 3]
) -> TensorType[0]:
"""
Normal consistency loss proposed in monosdf - https://niujinshuchong.github.io/monosdf/
Enforces consistency between the volume rendered normal and the predicted monocular normal.
With both angluar and L1 loss. Eq 14 https://arxiv.org/pdf/2206.00665.pdf
Args:
normal_pred: volume rendered normal
normal_gt: monocular normal
"""
normal_gt = torch.nn.functional.normalize(normal_gt, p=2, dim=-1)
normal_pred = torch.nn.functional.normalize(normal_pred, p=2, dim=-1)
l1 = torch.abs(normal_pred - normal_gt).sum(dim=-1).mean()
cos = (1.0 - torch.sum(normal_pred * normal_gt, dim=-1)).mean()
return l1 + cos
class MiDaSMSELoss(nn.Module):
"""
data term from MiDaS paper
"""
def __init__(self, reduction_type: Literal["image", "batch"] = "batch"):
super().__init__()
self.reduction_type = reduction_type
# reduction here is different from the image/batch-based reduction. This is either "mean" or "sum"
self.mse_loss = MSELoss(reduction="none")
def forward(
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
) -> TensorType[0]:
"""
Args:
prediction: predicted depth map
target: ground truth depth map
mask: mask of valid pixels
Returns:
mse loss based on reduction function
"""
summed_mask = torch.sum(mask, (1, 2))
image_loss = torch.sum(self.mse_loss(prediction, target) * mask, (1, 2))
image_loss = masked_reduction(image_loss, summed_mask, self.reduction_type)
return image_loss
# losses based on https://github.com/autonomousvision/monosdf/blob/main/code/model/loss.py
class GradientLoss(nn.Module):
"""
multiscale, scale-invariant gradient matching term to the disparity space.
This term biases discontinuities to be sharp and to coincide with discontinuities in the ground truth
More info here https://arxiv.org/pdf/1907.01341.pdf Equation 11
"""
def __init__(self, scales: int = 4, reduction_type: Literal["image", "batch"] = "batch"):
"""
Args:
scales: number of scales to use
reduction_type: either "batch" or "image"
"""
super().__init__()
self.reduction_type = reduction_type
self.__scales = scales
def forward(
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
) -> TensorType[0]:
"""
Args:
prediction: predicted depth map
target: ground truth depth map
mask: mask of valid pixels
Returns:
gradient loss based on reduction function
"""
total = 0
for scale in range(self.__scales):
step = pow(2, scale)
grad_loss = self.gradient_loss(
prediction[:, ::step, ::step],
target[:, ::step, ::step],
mask[:, ::step, ::step],
)
total += grad_loss
return total
def gradient_loss(
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
) -> TensorType[0]:
"""
multiscale, scale-invariant gradient matching term to the disparity space.
This term biases discontinuities to be sharp and to coincide with discontinuities in the ground truth
More info here https://arxiv.org/pdf/1907.01341.pdf Equation 11
Args:
prediction: predicted depth map
target: ground truth depth map
reduction: reduction function, either reduction_batch_based or reduction_image_based
Returns:
gradient loss based on reduction function
"""
summed_mask = torch.sum(mask, (1, 2))
diff = prediction - target
diff = torch.mul(mask, diff)
grad_x = torch.abs(diff[:, :, 1:] - diff[:, :, :-1])
mask_x = torch.mul(mask[:, :, 1:], mask[:, :, :-1])
grad_x = torch.mul(mask_x, grad_x)
grad_y = torch.abs(diff[:, 1:, :] - diff[:, :-1, :])
mask_y = torch.mul(mask[:, 1:, :], mask[:, :-1, :])
grad_y = torch.mul(mask_y, grad_y)
image_loss = torch.sum(grad_x, (1, 2)) + torch.sum(grad_y, (1, 2))
image_loss = masked_reduction(image_loss, summed_mask, self.reduction_type)
return image_loss
class ScaleAndShiftInvariantLoss(nn.Module):
"""
Scale and shift invariant loss as described in
"Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer"
https://arxiv.org/pdf/1907.01341.pdf
"""
def __init__(self, alpha: float = 0.5, scales: int = 4, reduction_type: Literal["image", "batch"] = "batch"):
"""
Args:
alpha: weight of the regularization term
scales: number of scales to use
reduction_type: either "batch" or "image"
"""
super().__init__()
self.__data_loss = MiDaSMSELoss(reduction_type=reduction_type)
self.__regularization_loss = GradientLoss(scales=scales, reduction_type=reduction_type)
self.__alpha = alpha
self.__prediction_ssi = None
@classmethod
def compute_scale_and_shift(
cls, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
):
"""
More info here: https://arxiv.org/pdf/2206.00665.pdf supplementary section A2 Depth Consistency Loss
This function computes scale/shift required to normalizes predicted depth map,
to allow for using normalized depth maps as input from monocular depth estimation networks.
These networks are trained such that they predict normalized depth maps.
Solves for scale/shift using a least squares approach with a closed form solution:
Based on:
https://github.com/autonomousvision/monosdf/blob/d9619e948bf3d85c6adec1a643f679e2e8e84d4b/code/model/loss.py#L7
Args:
prediction: predicted depth map
target: ground truth depth map
mask: mask of valid pixels
Returns:
scale and shift for depth prediction
"""
# system matrix: A = [[a_00, a_01], [a_10, a_11]]
a_00 = torch.sum(mask * prediction * prediction, (1, 2))
a_01 = torch.sum(mask * prediction, (1, 2))
a_11 = torch.sum(mask, (1, 2))
# right hand side: b = [b_0, b_1]
b_0 = torch.sum(mask * prediction * target, (1, 2))
b_1 = torch.sum(mask * target, (1, 2))
# solution: x = A^-1 . b = [[a_11, -a_01], [-a_10, a_00]] / (a_00 * a_11 - a_01 * a_10) . b
scale = torch.zeros_like(b_0)
shift = torch.zeros_like(b_1)
det = a_00 * a_11 - a_01 * a_01
valid = det.nonzero()
scale[valid] = (a_11[valid] * b_0[valid] - a_01[valid] * b_1[valid]) / det[valid]
shift[valid] = (-a_01[valid] * b_0[valid] + a_00[valid] * b_1[valid]) / det[valid]
return scale, shift
def forward(
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
) -> TensorType[0]:
"""
Args:
prediction: predicted depth map (unnormalized)
target: ground truth depth map (normalized)
mask: mask of valid pixels
Returns:
scale and shift invariant loss
"""
scale, shift = ScaleAndShiftInvariantLoss.compute_scale_and_shift(prediction, target, mask)
self.__prediction_ssi = scale.view(-1, 1, 1) * prediction + shift.view(-1, 1, 1)
total = self.__data_loss(self.__prediction_ssi, target, mask)
if self.__alpha > 0:
total += self.__alpha * self.__regularization_loss(self.__prediction_ssi, target, mask)
return total
def __get_prediction_ssi(self):
"""
scale and shift invariant prediction
from https://arxiv.org/pdf/1907.01341.pdf equation 1
"""
return self.__prediction_ssi
prediction_ssi = property(__get_prediction_ssi)
+4 -3
View File
@@ -37,6 +37,7 @@ from torchtyping import TensorType
from typing_extensions import Literal
from nerfstudio.cameras.rays import RaySamples
from nerfstudio.utils import colors
from nerfstudio.utils.math import components_from_spherical_harmonics, safe_normalize
BACKGROUND_COLOR_OVERRIDE: Optional[TensorType[3]] = None
@@ -70,7 +71,7 @@ class RGBRenderer(nn.Module):
cls,
rgb: TensorType["bs":..., "num_samples", 3],
weights: TensorType["bs":..., "num_samples", 1],
background_color: Union[Literal["random", "black", "last_sample"], TensorType[3]] = "random",
background_color: Union[Literal["random", "white", "black", "last_sample"], TensorType[3]] = "random",
ray_indices: Optional[TensorType["num_samples"]] = None,
num_rays: Optional[int] = None,
) -> TensorType["bs":..., 3]:
@@ -102,8 +103,8 @@ class RGBRenderer(nn.Module):
background_color = rgb[..., -1, :]
if background_color == "random":
background_color = torch.rand_like(comp_rgb).to(rgb.device)
if background_color == "black":
background_color = torch.zeros_like(comp_rgb).to(rgb.device)
if isinstance(background_color, str) and background_color in colors.COLORS_DICT:
background_color = colors.COLORS_DICT[background_color].to(rgb.device)
assert isinstance(background_color, torch.Tensor)
comp_rgb = comp_rgb + background_color.to(weights.device) * (1.0 - accumulated_weight)
+2 -1
View File
@@ -70,7 +70,8 @@ class DepthNerfactoModel(NerfactoModel):
def get_outputs(self, ray_bundle: RayBundle):
outputs = super().get_outputs(ray_bundle)
outputs["directions_norm"] = ray_bundle.metadata["directions_norm"]
if ray_bundle.metadata is not None and "directions_norm" in ray_bundle.metadata:
outputs["directions_norm"] = ray_bundle.metadata["directions_norm"]
return outputs
def get_metrics_dict(self, outputs, batch):
+2 -6
View File
@@ -46,7 +46,7 @@ from nerfstudio.model_components.renderers import (
RGBRenderer,
)
from nerfstudio.models.base_model import Model, ModelConfig
from nerfstudio.utils import colormaps, colors
from nerfstudio.utils import colormaps
@dataclass
@@ -123,11 +123,7 @@ class NGPModel(Model):
)
# renderers
background_color = "random"
if self.config.background_color in ["white", "black"]:
background_color = colors.COLORS_DICT[self.config.background_color]
self.renderer_rgb = RGBRenderer(background_color=background_color)
self.renderer_rgb = RGBRenderer(background_color=self.config.background_color)
self.renderer_accumulation = AccumulationRenderer()
self.renderer_depth = DepthRenderer(method="expected")
+1 -1
View File
@@ -68,7 +68,7 @@ class NerfactoModelConfig(ModelConfig):
"""How far along the ray to start sampling."""
far_plane: float = 1000.0
"""How far along the ray to stop sampling."""
background_color: Literal["random", "last_sample"] = "last_sample"
background_color: Literal["random", "last_sample", "black", "white"] = "last_sample"
"""Whether to randomize the background color."""
num_levels: int = 16
"""Number of levels of the hashmap for the base mlp."""
+11 -3
View File
@@ -50,6 +50,7 @@ from nerfstudio.model_components.renderers import (
RGBRenderer,
)
from nerfstudio.model_components.scene_colliders import NearFarCollider
from nerfstudio.model_components.shaders import NormalsShader
from nerfstudio.models.base_model import Model
from nerfstudio.models.nerfacto import NerfactoModel, NerfactoModelConfig
@@ -63,7 +64,7 @@ class NerfplayerNerfactoModelConfig(NerfactoModelConfig):
"""How far along the ray to start sampling."""
far_plane: float = 1000.0
"""How far along the ray to stop sampling."""
background_color: Literal["random", "last_sample"] = "random"
background_color: Literal["random", "last_sample", "black", "white"] = "random"
"""Whether to randomize the background color. (Random is reported to be better on DyCheck.)"""
num_levels: int = 16
"""Hashing grid parameter."""
@@ -162,6 +163,9 @@ class NerfplayerNerfactoModel(NerfactoModel):
self.renderer_depth = DepthRenderer(method="expected") # for depth loss
self.renderer_normals = NormalsRenderer()
# shaders
self.normals_shader = NormalsShader()
# losses
self.rgb_loss = MSELoss()
@@ -192,8 +196,12 @@ class NerfplayerNerfactoModel(NerfactoModel):
}
if self.config.predict_normals:
outputs["normals"] = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights)
outputs["pred_normals"] = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights)
outputs["normals"] = self.normals_shader(
self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights)
)
outputs["pred_normals"] = self.normals_shader(
self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights)
)
# These use a lot of GPU memory, so we avoid storing them for eval.
if self.training:
+2 -10
View File
@@ -42,7 +42,6 @@ from nerfstudio.model_components.renderers import (
)
from nerfstudio.models.base_model import Model
from nerfstudio.models.instant_ngp import InstantNGPModelConfig, NGPModel
from nerfstudio.utils import colors
@dataclass
@@ -128,13 +127,6 @@ class NerfplayerNGPModel(NGPModel):
) # need to update the density_fn later during forward (for input time)
# renderers
self.train_background_color = self.config.train_background_color
self.eval_background_color = self.config.eval_background_color
if self.config.train_background_color in ["white", "black"]:
self.train_background_color = colors.COLORS_DICT[self.config.train_background_color]
if self.config.eval_background_color in ["white", "black"]:
self.eval_background_color = colors.COLORS_DICT[self.config.eval_background_color]
self.renderer_rgb = RGBRenderer() # will update bgcolor later during forward
self.renderer_accumulation = AccumulationRenderer()
self.renderer_depth = DepthRenderer(method="expected")
@@ -175,9 +167,9 @@ class NerfplayerNGPModel(NGPModel):
# update bgcolor in the renderer; usually random color for training and fixed color for inference
if self.training:
self.renderer_rgb.background_color = self.train_background_color
self.renderer_rgb.background_color = self.config.train_background_color
else:
self.renderer_rgb.background_color = self.eval_background_color
self.renderer_rgb.background_color = self.config.eval_background_color
rgb = self.renderer_rgb(
rgb=field_outputs[FieldHeadNames.RGB],
weights=weights,
+1 -10
View File
@@ -604,14 +604,6 @@ def colmap_to_json(
cameras = read_cameras_binary(cameras_path)
images = read_images_binary(images_path)
# Images were renamed to frame_{i:05d}.{ext} and
# the filenames needs to be replaced in the transforms.json as well
original_filenames = [x.name for x in images.values()]
# Sort was used in nerfstudio.process_data.process_data_utils:get_image_filenames
original_filenames.sort()
# Build the map to the new filenames
filename_map = {name: f"frame_{i+1:05d}{os.path.splitext(name)[-1]}" for i, name in enumerate(original_filenames)}
# Only supports one camera
camera_params = cameras[1].params
@@ -627,8 +619,7 @@ def colmap_to_json(
c2w = c2w[np.array([1, 0, 2, 3]), :]
c2w[2, :] *= -1
name = filename_map[im_data.name]
name = Path(f"./images/{name}")
name = Path(f"./images/{im_data.name}")
frame = {
"file_path": name.as_posix(),
+26
View File
@@ -19,6 +19,7 @@ from typing import Tuple
import torch
from torchtyping import TensorType
from typing_extensions import Literal
_USE_NERFACC = True
@@ -285,3 +286,28 @@ def safe_normalize(
Normalized vectors.
"""
return vectors / (torch.norm(vectors, dim=-1, keepdim=True) + eps)
def masked_reduction(
input_tensor: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"], reduction_type: Literal["image", "batch"]
):
"""
Whether to consolidate the input_tensor across the batch or across the image
Args:
input_tensor: input tensor
mask: mask tensor
reduction_type: either "batch" or "image"
Returns:
input_tensor: reduced input_tensor
"""
if reduction_type == "batch":
# avoid division by 0 (if sum(M) = sum(sum(mask)) = 0: sum(image_loss) = 0)
divisor = torch.sum(mask)
input_tensor = 0 if divisor == 0 else torch.sum(input_tensor) / divisor
elif reduction_type == "image":
# avoid division by 0 (if M = sum(mask) = 0: image_loss = 0)
valid = mask.nonzero()
input_tensor[valid] = input_tensor[valid] / mask[valid]
input_tensor = torch.mean(input_tensor)
return input_tensor
+1 -1
View File
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "nerfstudio"
version = "0.1.17"
version = "0.1.18"
description = "All-in-one repository for state-of-the-art NeRFs"
readme = "README.md"
license = { text="Apache 2.0"}
View File
File mode changed.
+1 -1
View File
@@ -28,7 +28,7 @@ fi
method_opts=()
if [ "$method_name" = "nerfacto" ]; then
# https://github.com/nerfstudio-project/nerfstudio/issues/806#issuecomment-1284327844
method_opts=(--pipeline.model.near-plane 2. --pipeline.model.far-plane 6. --pipeline.datamanager.camera-optimizer.mode off --pipeline.model.use-average-appearance-embedding False)
method_opts=(--pipeline.model.background-color white --pipeline.model.near-plane 2. --pipeline.model.far-plane 6. --pipeline.datamanager.camera-optimizer.mode off --pipeline.model.use-average-appearance-embedding False)
fi
shift $((OPTIND-1))
Executable → Regular
+199 -84
View File
@@ -10,27 +10,29 @@ import shutil
import stat
import subprocess
import sys
from typing import List, Union
from typing import List, Optional, Union
import tyro
from rich.console import Console
from rich.prompt import Confirm
from typing_extensions import Literal, assert_never
from typing_extensions import get_args as typing_get_args
if sys.version_info < (3, 10):
import importlib_metadata
else:
from importlib import metadata as importlib_metadata
ConfigureMode = Literal["install", "uninstall"]
ShellType = Literal["zsh", "bash"]
CONSOLE = Console(width=120)
HEADER_LINE = "# Source nerfstudio autocompletions."
ENTRYPOINTS = [
"ns-install-cli",
"ns-process-data",
"ns-download-data",
"ns-train",
"ns-eval",
"ns-render",
"ns-dev-test",
]
def _get_all_entry_points() -> List[str]:
entry_points = importlib_metadata.distribution("nerfstudio").entry_points
return [x.name for x in entry_points]
def _check_tyro_cli(script_path: pathlib.Path) -> bool:
@@ -127,6 +129,62 @@ def _exclamation() -> str:
return random.choice(["Cool", "Nice", "Neat", "Great", "Exciting", "Excellent", "Ok"]) + "!"
def _get_deactivate_script(commands: List[str], shell: Optional[ShellType], add_header=True) -> str:
if shell is None:
# Install the universal script
result_script = []
for shell_type in typing_get_args(ShellType):
result_script.append(f'if [ -n "${shell_type.upper()}_VERSION" ]; then')
result_script.append(_get_deactivate_script(commands, shell_type, add_header=False))
result_script.append("fi")
source_lines = "\n".join(result_script)
elif shell == "zsh":
source_lines = "\n".join([f"unset '_comps[{command}]' &> /dev/null" for command in commands])
elif shell == "bash":
source_lines = "\n".join([f"complete -r {command} &> /dev/null" for command in commands])
else:
assert_never(shell)
if add_header:
source_lines = f"\n{HEADER_LINE}\n{source_lines}"
return source_lines
def _get_source_script(completions_dir: pathlib.Path, shell: Optional[ShellType], add_header=True) -> str:
if shell is None:
# Install the universal script
result_script = []
for shell_type in typing_get_args(ShellType):
result_script.append(f'if [ -n "${shell_type.upper()}_VERSION" ]; then')
result_script.append(_get_source_script(completions_dir, shell_type, add_header=False))
result_script.append("fi")
source_lines = "\n".join(result_script)
elif shell == "zsh":
source_lines = "\n".join(
[
"if ! command -v compdef &> /dev/null; then",
" autoload -Uz compinit",
" compinit",
"fi",
f"source {completions_dir / 'setup.zsh'}",
]
)
elif shell == "bash":
source_lines = "\n".join(
[
f"source {completions_dir / 'setup.bash'}",
]
)
else:
assert_never(shell)
if add_header:
source_lines = f"\n{HEADER_LINE}\n{source_lines}"
return source_lines
def _update_rc(
completions_dir: pathlib.Path,
mode: ConfigureMode,
@@ -141,36 +199,13 @@ def _update_rc(
"""
# Install or uninstall `source_line`.
header_line = "# Source nerfstudio autocompletions."
if shell == "zsh":
source_lines = "\n".join(
[
"",
header_line,
"if ! command -v compdef &> /dev/null; then",
" autoload -Uz compinit",
" compinit",
"fi",
f"source {completions_dir / 'setup.zsh'}",
]
)
elif shell == "bash":
source_lines = "\n".join(
[
"",
header_line,
f"source {completions_dir / 'setup.bash'}",
]
)
else:
assert_never(shell)
source_lines = _get_source_script(completions_dir, shell)
rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
# Always try to uninstall previous completions.
rc_source = rc_path.read_text()
while header_line in rc_source:
before_install, _, after_install = rc_source.partition(header_line)
while HEADER_LINE in rc_source:
before_install, _, after_install = rc_source.partition(HEADER_LINE)
source_file, _, after_install = after_install.partition("\nsource ")[2].partition("\n")
assert source_file.endswith(f"/completions/setup.{shell}")
rc_source = before_install + after_install
@@ -189,6 +224,108 @@ def _update_rc(
assert mode == "uninstall"
def _update_conda_scripts(
commands: List[str],
completions_dir: pathlib.Path,
mode: ConfigureMode,
) -> None:
"""Try to add a `source /.../completions/setup.{shell}` line automatically to conda's activation scripts.
Args:
completions_dir: Path to location of this script.
mode: Install or uninstall completions.
"""
# Install or uninstall `source_line`.
activate_source_lines = _get_source_script(completions_dir, None)
deactivate_source_lines = _get_deactivate_script(commands, None)
conda_path = pathlib.Path(os.environ["CONDA_PREFIX"])
activate_path = conda_path / "etc/conda/activate.d/nerfstudio_activate.sh"
deactivate_path = conda_path / "etc/conda/deactivate.d/nerfstudio_deactivate.sh"
if mode == "uninstall":
if activate_path.exists():
os.remove(activate_path)
if deactivate_path.exists():
os.remove(deactivate_path)
CONSOLE.log(f":broom: Existing completions uninstalled from {conda_path}.")
elif mode == "install":
# Install completions.
activate_path.parent.mkdir(exist_ok=True, parents=True)
deactivate_path.parent.mkdir(exist_ok=True, parents=True)
with activate_path.open("w+", encoding="utf8") as f:
f.write(activate_source_lines)
with deactivate_path.open("w+", encoding="utf8") as f:
f.write(deactivate_source_lines)
CONSOLE.log(
f":person_gesturing_ok: Completions installed to {conda_path}. {_exclamation()} Reactivate the environment"
" to try them out."
)
else:
assert_never(mode)
def _get_conda_path() -> Optional[pathlib.Path]:
"""
Returns the path to the conda environment if
the nerfstudio package is installed in one.
"""
conda_path = None
if "CONDA_PREFIX" in os.environ:
# Conda is active, we will check if the Nerfstudio is installed in the conda env.
distribution = importlib_metadata.distribution("nerfstudio")
if str(distribution.locate_file("nerfstudio")).startswith(os.environ["CONDA_PREFIX"]):
conda_path = pathlib.Path(os.environ["CONDA_PREFIX"])
return conda_path
def _generate_completions_files(
completions_dir: pathlib.Path,
scripts_dir: pathlib.Path,
shells_supported: List[ShellType],
shells_found: List[ShellType],
) -> None:
# Set to True to install completions for scripts as well.
include_scripts = False
# Find tyro CLIs.
script_paths = list(filter(_check_tyro_cli, scripts_dir.glob("**/*.py"))) if include_scripts else []
script_names = tuple(p.name for p in script_paths)
assert len(set(script_names)) == len(script_names)
# Get existing completion files.
existing_completions = set()
for shell in shells_supported:
target_dir = completions_dir / shell
if target_dir.exists():
existing_completions |= set(target_dir.glob("*"))
# Get all entry_points.
entry_points = _get_all_entry_points()
# Run generation jobs.
concurrent_executor = concurrent.futures.ThreadPoolExecutor()
with CONSOLE.status("[bold]:writing_hand: Generating completions...", spinner="bouncingBall"):
completion_paths = list(
concurrent_executor.map(
lambda path_or_entrypoint_and_shell: _generate_completion(
path_or_entrypoint_and_shell[0], path_or_entrypoint_and_shell[1], completions_dir
),
itertools.product(script_paths + entry_points, shells_found),
)
)
# Delete obsolete completion files.
for unexpected_path in set(p.absolute() for p in existing_completions) - set(
p.absolute() for p in completion_paths
):
if unexpected_path.is_dir():
shutil.rmtree(unexpected_path)
elif unexpected_path.exists():
unexpected_path.unlink()
CONSOLE.log(f":broom: Deleted {unexpected_path}.")
def main(mode: ConfigureMode = "install") -> None:
"""Main script.
@@ -201,16 +338,24 @@ def main(mode: ConfigureMode = "install") -> None:
CONSOLE.log("[bold red]$HOME is not set. Exiting.")
return
# Get conda path if in conda environment.
conda_path = _get_conda_path()
# Try to locate the user's bashrc or zshrc.
shells_supported: List[ShellType] = ["zsh", "bash"]
shells_found: List[ShellType] = []
for shell in shells_supported:
rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
if not rc_path.exists():
CONSOLE.log(f":person_shrugging: {rc_path.name} not found, skipping.")
else:
CONSOLE.log(f":mag: Found {rc_path.name}!")
shells_found.append(shell)
shells_supported: List[ShellType] = list(typing_get_args(ShellType))
if conda_path is not None:
# Running in conda; we have to support all shells.
shells_found = shells_supported
CONSOLE.log(f":mag: Detected conda environment {conda_path}!")
else:
shells_found: List[ShellType] = []
for shell in shells_supported:
rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc"
if not rc_path.exists():
CONSOLE.log(f":person_shrugging: {rc_path.name} not found, skipping.")
else:
CONSOLE.log(f":mag: Found {rc_path.name}!")
shells_found.append(shell)
# Get scripts/ directory.
completions_dir = pathlib.Path(__file__).absolute().parent
@@ -230,48 +375,18 @@ def main(mode: ConfigureMode = "install") -> None:
else:
CONSOLE.log(f":heavy_check_mark: No existing completions at: {target_dir}.")
elif mode == "install":
# Set to True to install completions for scripts as well.
include_scripts = False
# Find tyro CLIs.
script_paths = list(filter(_check_tyro_cli, scripts_dir.glob("**/*.py"))) if include_scripts else []
script_names = tuple(p.name for p in script_paths)
assert len(set(script_names)) == len(script_names)
# Get existing completion files.
existing_completions = set()
for shell in shells_supported:
target_dir = completions_dir / shell
if target_dir.exists():
existing_completions |= set(target_dir.glob("*"))
# Run generation jobs.
concurrent_executor = concurrent.futures.ThreadPoolExecutor()
with CONSOLE.status("[bold]:writing_hand: Generating completions...", spinner="bouncingBall"):
completion_paths = list(
concurrent_executor.map(
lambda path_or_entrypoint_and_shell: _generate_completion(
path_or_entrypoint_and_shell[0], path_or_entrypoint_and_shell[1], completions_dir
),
itertools.product(script_paths + ENTRYPOINTS, shells_found),
)
)
# Delete obsolete completion files.
for unexpected_path in set(p.absolute() for p in existing_completions) - set(
p.absolute() for p in completion_paths
):
if unexpected_path.is_dir():
shutil.rmtree(unexpected_path)
elif unexpected_path.exists():
unexpected_path.unlink()
CONSOLE.log(f":broom: Deleted {unexpected_path}.")
_generate_completions_files(completions_dir, scripts_dir, shells_supported, shells_found)
else:
assert_never(mode)
# Install or uninstall from bashrc/zshrc.
for shell in shells_found:
_update_rc(completions_dir, mode, shell)
if conda_path is not None:
# In conda environment we add the completitions activation scripts.
commands = _get_all_entry_points()
_update_conda_scripts(commands, completions_dir, mode)
else:
# Install or uninstall from bashrc/zshrc.
for shell in shells_found:
_update_rc(completions_dir, mode, shell)
CONSOLE.print("[bold]All done![/bold]")
Executable → Regular
View File
File mode changed.
Executable → Regular
View File
File mode changed.
View File
File mode changed.
Executable → Regular
View File
File mode changed.
Executable → Regular
View File
File mode changed.
Executable → Regular
View File
File mode changed.
Executable → Regular
View File
File mode changed.
View File
File mode changed.