From edef6d62f675c3357f5b856960c10c2862add6de Mon Sep 17 00:00:00 2001 From: machenmusik Date: Tue, 31 Jan 2023 21:52:42 -0500 Subject: [PATCH 01/30] bump nerfacc to 0.3.4 (#1322) Co-authored-by: Chen, Michael --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 10942cf21..8b1700d4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "mediapy>=1.1.0", "msgpack>=1.0.4", "msgpack_numpy>=0.4.8", - "nerfacc==0.3.3", + "nerfacc==0.3.4", "open3d>=0.16.0", "opencv-python==4.6.0.66", "Pillow>=9.3.0", From f5f424095a05ce565249955f5919f4c06e3c7319 Mon Sep 17 00:00:00 2001 From: machenmusik Date: Tue, 31 Jan 2023 22:27:05 -0500 Subject: [PATCH 02/30] Document localhost URL (fixes #1323) (#1324) Document localhost URL Co-authored-by: Chen, Michael --- docs/developer_guides/viewer/viewer_overview.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/developer_guides/viewer/viewer_overview.md b/docs/developer_guides/viewer/viewer_overview.md index b80463e75..4e10319b6 100644 --- a/docs/developer_guides/viewer/viewer_overview.md +++ b/docs/developer_guides/viewer/viewer_overview.md @@ -50,6 +50,10 @@ yarn install yarn start ``` +The local webserver runs on port 4000 by default, +so when `ns-train` is running, you can connect to the viewer locally at +[http://localhost:4000/?websocket_url=ws://localhost:7007](http://localhost:4000/?websocket_url=ws://localhost:7007) + #### Running the Bridge Server Manually The viewer bridge server runs on the same machine that you use for training. The training code will connect to this server with a lightweight TCP connection using ZMQ. The training job will launch the viewer bridge server is you specify `--viewer.launch-bridge-server` in the terminal. Otherwise, you can launch the bridge server manually with the following script. From 11e7fa87e04f90ad94598d6b41cacfa91fff4751 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Wed, 1 Feb 2023 10:29:08 -0800 Subject: [PATCH 03/30] Add Stable Diffusion helper code (#1320) * Add stable diffusion * Add dependencies fix lint * Update types * Update comment * fix comments --------- Co-authored-by: David McAllister Co-authored-by: Abhik Ahuja --- nerfstudio/generative/__init__.py | 13 + nerfstudio/generative/stable_diffusion.py | 336 ++++++++++++++++++++++ pyproject.toml | 8 + 3 files changed, 357 insertions(+) create mode 100644 nerfstudio/generative/__init__.py create mode 100644 nerfstudio/generative/stable_diffusion.py diff --git a/nerfstudio/generative/__init__.py b/nerfstudio/generative/__init__.py new file mode 100644 index 000000000..d926cf95c --- /dev/null +++ b/nerfstudio/generative/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nerfstudio/generative/stable_diffusion.py b/nerfstudio/generative/stable_diffusion.py new file mode 100644 index 000000000..af3aa2196 --- /dev/null +++ b/nerfstudio/generative/stable_diffusion.py @@ -0,0 +1,336 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Stable Diffusion helpers""" + +# Modified from https://github.com/ashawkey/stable-dreamfusion/blob/main/nerf/sd.py + +import sys +from pathlib import Path +from typing import List, Optional, Tuple, Union + +import mediapy +import numpy as np +import torch +import torch.nn.functional as F +import tyro +from rich.console import Console +from torch import nn +from torchtyping import TensorType + +CONSOLE = Console(width=120) + +try: + from diffusers import PNDMScheduler, StableDiffusionPipeline + from transformers import logging + +except ImportError: + CONSOLE.print("[bold red]Missing Stable Diffusion packages.") + CONSOLE.print(r"Install using [yellow]pip install nerfstudio\[gen][/yellow]") + CONSOLE.print(r"or [yellow]pip install -e .\[gen][/yellow] if installing from source.") + sys.exit(1) + +logging.set_verbosity_error() +IMG_DIM = 512 +CONST_SCALE = 0.18215 + +SD_SOURCE = "runwayml/stable-diffusion-v1-5" +CLIP_SOURCE = "openai/clip-vit-large-patch14" + + +class StableDiffusion(nn.Module): + """Stable Diffusion implementation + + Args: + device: device to use + num_train_timesteps: number of training timesteps + + """ + + def __init__(self, device: Union[torch.device, str], num_train_timesteps: int = 1000) -> None: + super().__init__() + + self.device = device + self.num_train_timesteps = num_train_timesteps + + self.min_step = int(self.num_train_timesteps * 0.02) + self.max_step = int(self.num_train_timesteps * 0.98) + + self.scheduler = PNDMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + num_train_timesteps=self.num_train_timesteps, + ) + self.alphas = self.scheduler.alphas_cumprod.to(self.device) # type: ignore + + pipe = StableDiffusionPipeline.from_pretrained(SD_SOURCE, torch_dtype=torch.float16) + assert pipe is not None + pipe = pipe.to(self.device) + + pipe.enable_attention_slicing() + + self.tokenizer = pipe.tokenizer + self.unet = pipe.unet + self.text_encoder = pipe.text_encoder + self.auto_encoder = pipe.vae + + self.unet.to(memory_format=torch.channels_last) + + CONSOLE.print("Stable Diffusion loaded!") + + def get_text_embeds( + self, prompt: Union[str, List[str]], negative_prompt: Union[str, List[str]] + ) -> TensorType[2, "max_length", "embed_dim"]: + """Get text embeddings for prompt and negative prompt + + Args: + prompt: Prompt text + negative_prompt: Negative prompt text + + Returns: + Text embeddings + """ + + # Tokenize text and get embeddings + text_input = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + + with torch.no_grad(): + text_embeddings = self.text_encoder(text_input.input_ids.to(self.device))[0] + + # Do the same for unconditional embeddings + uncond_input = self.tokenizer( + negative_prompt, padding="max_length", max_length=self.tokenizer.model_max_length, return_tensors="pt" + ) + + with torch.no_grad(): + uncond_embeddings = self.text_encoder(uncond_input.input_ids.to(self.device))[0] + + # Cat for final embeddings + text_embeddings = torch.cat([uncond_embeddings, text_embeddings]) + + return text_embeddings + + def sds_loss( + self, + text_embeddings: TensorType["N", "max_length", "embed_dim"], + image: TensorType["BS", 3, "H", "W"], + guidance_scale: float = 100.0, + ) -> Tuple[torch.Tensor, TensorType["BS", 4, "H", "W"], TensorType["BS", 4, "H", "W"]]: + """Score Distilation Sampling loss proposed in DreamFusion paper (https://dreamfusion3d.github.io/) + + Args: + text_embeddings: Text embeddings + image: Rendered image + guidance_scale: How much to weigh the guidance + + Returns: + A tuple of Loss, latent, and gradient + """ + image = F.interpolate(image, (IMG_DIM, IMG_DIM), mode="bilinear") + t = torch.randint(self.min_step, self.max_step + 1, [1], dtype=torch.long, device=self.device) + latents = self.imgs_to_latent(image) + + # predict the noise residual with unet, NO grad! + with torch.no_grad(): + # add noise + noise = torch.randn_like(latents) + latents_noisy = self.scheduler.add_noise(latents, noise, t) # type: ignore + # pred noise + latent_model_input = torch.cat([latents_noisy] * 2) + noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings).sample + + # perform guidance + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # w(t), sigma_t^2 + w = 1 - self.alphas[t] + + grad = w * (noise_pred - noise) + grad = torch.nan_to_num(grad) + + sds_mse = torch.mean(torch.nan_to_num(torch.square(noise_pred - noise))) + + return sds_mse, latents, grad + + def produce_latents( + self, + text_embeddings: TensorType["N", "max_length", "embed_dim"], + height: int = IMG_DIM, + width: int = IMG_DIM, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + latents: Optional[TensorType["BS", 4, "H", "W"]] = None, + ) -> TensorType["BS", 4, "H", "W"]: + """Produce latents for a given text embedding + + Args: + text_embeddings: Text embeddings + height: Height of the image + width: Width of the image + num_inference_steps: Number of inference steps + guidance_scale: How much to weigh the guidance + latents: Latents to start with + + Returns: + Latents + """ + + if latents is None: + latents = torch.randn( + (text_embeddings.shape[0] // 2, self.unet.in_channels, height // 8, width // 8), device=self.device + ) + + self.scheduler.set_timesteps(num_inference_steps) # type: ignore + + with torch.autocast("cuda"): + for t in self.scheduler.timesteps: # type: ignore + assert latents is not None + # expand the latents if we are doing classifier-free guidance to avoid doing two forward passes. + latent_model_input = torch.cat([latents] * 2) + + # predict the noise residual + with torch.no_grad(): + noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings)["sample"] + + # perform guidance + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents)["prev_sample"] # type: ignore + return latents + + def latents_to_img(self, latents: TensorType["BS", 4, "H", "W"]) -> TensorType["BS", 3, "H", "W"]: + """Convert latents to images + + Args: + latents: Latents to convert + + Returns: + Images + """ + + latents = 1 / CONST_SCALE * latents + + with torch.no_grad(): + imgs = self.auto_encoder.decode(latents).sample + + imgs = (imgs / 2 + 0.5).clamp(0, 1) + + return imgs + + def imgs_to_latent(self, imgs: TensorType["BS", 3, "H", "W"]) -> TensorType["BS", 4, "H", "W"]: + """Convert images to latents + + Args: + imgs: Images to convert + + Returns: + Latents + """ + imgs = 2 * imgs - 1 + + posterior = self.auto_encoder.encode(imgs).latent_dist + latents = posterior.sample() * CONST_SCALE + + return latents + + def prompt_to_img( + self, + prompts: Union[str, List[str]], + negative_prompts: Union[str, List[str]] = "", + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + latents=None, + ) -> np.ndarray: + """Generate an images from a prompts. + + Args: + prompts: The prompt to generate an image from. + negative_prompts: The negative prompt to generate an image from. + num_inference_steps: The number of inference steps to perform. + guidance_scale: The scale of the guidance. + latents: The latents to start from, defaults to random. + + Returns: + The generated image. + """ + + prompts = [prompts] if isinstance(prompts, str) else prompts + negative_prompts = [negative_prompts] if isinstance(negative_prompts, str) else negative_prompts + text_embeddings = self.get_text_embeds(prompts, negative_prompts) + latents = self.produce_latents( + text_embeddings, + height=IMG_DIM, + width=IMG_DIM, + latents=latents, + num_inference_steps=num_inference_steps, + guidance_scale=guidance_scale, + ) # [1, 4, resolution, resolution] + + diffused_img = self.latents_to_img(latents.half()) + diffused_img = diffused_img.detach().cpu().permute(0, 2, 3, 1).numpy() + diffused_img = (diffused_img * 255).round().astype("uint8") + + return diffused_img + + def forward( + self, prompts, negative_prompts="", num_inference_steps=50, guidance_scale=7.5, latents=None + ) -> np.ndarray: + """Generate an image from a prompt. + + Args: + prompts: The prompt to generate an image from. + negative_prompts: The negative prompt to generate an image from. + num_inference_steps: The number of inference steps to perform. + guidance_scale: The scale of the guidance. + latents: The latents to start from, defaults to random. + + Returns: + The generated image. + """ + return self.prompt_to_img(prompts, negative_prompts, num_inference_steps, guidance_scale, latents) + + +def generate_image( + prompt: str, negative: str = "", seed: int = 0, steps: int = 50, save_path: Path = Path("test_sd.png") +): + """Generate an image from a prompt using Stable Diffusion. + + Args: + prompt: The prompt to use. + negative: The negative prompt to use. + seed: The random seed to use. + steps: The number of steps to use. + save_path: The path to save the image to. + """ + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + cuda_device = torch.device("cuda") + with torch.no_grad(): + sd = StableDiffusion(cuda_device) + imgs = sd.prompt_to_img(prompt, negative, steps) + mediapy.write_image(str(save_path), imgs[0]) + + +if __name__ == "__main__": + tyro.cli(generate_image) diff --git a/pyproject.toml b/pyproject.toml index 8b1700d4d..ca44d209e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,6 +59,14 @@ dependencies = [ [project.optional-dependencies] + +# Generative related dependencies +gen = [ + "diffusers==0.9.0", + "transformers==4.24.0", +] + + # Development packages dev = [ "black[jupyter]==22.3.0", From 9966c43811776d4de129068b89f105de18d8a57d Mon Sep 17 00:00:00 2001 From: Nicolas Zunhammer Date: Wed, 1 Feb 2023 21:02:10 +0100 Subject: [PATCH 04/30] [Fix] Docker not supporting older CUDA architectures (#1328) Fix docker not supporting older CUDA architectures. Co-authored-by: Nicolas Zunhammer Co-authored-by: Matthew Tancik --- Dockerfile | 3 ++- docs/quickstart/installation.md | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c2e3ab82b..94587f550 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,8 @@ ENV DEBIAN_FRONTEND=noninteractive ## Set timezone as it is required by some packages. ENV TZ=Europe/Berlin ## CUDA architectures, required by tiny-cuda-nn. -ENV TCNN_CUDA_ARCHITECTURES=86 +## NOTE: All commonly used GPU architectures are included and supported here. To speedup the image build process remove all architectures but the one of your explicit GPU. Find details here: https://developer.nvidia.com/cuda-gpus (8.6 translates to 86 in the line below) or in the docs. +ENV TCNN_CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37 ## CUDA Home, required to find CUDA in some packages. ENV CUDA_HOME="/usr/local/cuda" diff --git a/docs/quickstart/installation.md b/docs/quickstart/installation.md index ae6a17cc9..97c63aac8 100644 --- a/docs/quickstart/installation.md +++ b/docs/quickstart/installation.md @@ -95,13 +95,28 @@ docker run --gpus all -v /folder/of/your/data:/workspace/ -v /home//. ``` ### Note - The container works on Linux and Windows, depending on your OS some additional setup steps might be required to provide access to your GPU inside containers. -- Paths on Windows use backslash '\\\\' while unix based systems use a frontslash '/' for paths, where backslashes might require an escape character depending on where they are used (e.g. C:\\\\folder1\\\\folder2...). Ensure to use the correct paths when mounting folders or providing paths as parameters. +- Paths on Windows use backslash '\\' while unix based systems use a frontslash '/' for paths, where backslashes might require an escape character depending on where they are used (e.g. C:\\\\folder1\\\\folder2...). Alternatively, mounts can be quoted (e.g. ```-v 'C:\local_folder:/docker_folder'```). Ensure to use the correct paths when mounting folders or providing paths as parameters. +- Always use full paths, relative paths are known to create issues when being used in mounts into docker. - Everything inside the container, what is not in a mounted folder (workspace in the above example), will be permanently removed after destroying the container. Always do all your tasks and output folder in workdir! - The user inside the container is called 'user' and is mapped to the local user with ID 1000 (usually the first non-root user on Linux systems). - The container currently is based on nvidia/cuda:11.7.1-devel-ubuntu22.04, consequently it comes with CUDA 11.7 which must be supported by the nvidia driver. No local CUDA installation is required or will be affected by using the docker image. - The docker image (respectively Ubuntu 22.04) comes with Python3.10, no older version of Python is installed. - If you call the container with commands directly, you still might want to add the interactive terminal ('-it') flag to get live log outputs of the nerfstudio scripts. In case the container is used in an automated environment the flag should be discarded. +- The current version of docker is built for multi-architecture (CUDA architectures) use. The target architecture must be defined at build time for tinyCUDNN to be able to compile properly. If your GPU architecture is not covered by the following table you need to replace the number in the line ```ENV TCNN_CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37``` to you specific architecture. It also is a good idea to remove all architectures but yours (e.g. ```ENV TCNN_CUDA_ARCHITECTURES=86```) to speedup the docker build a lot. +**Currently supported CUDA architectures in the docker image** + +GPU | CUDA arch +-- | -- +H100 | 90 +40X0 | 89 +30X0 | 86 +A100 | 80 +20X0 | 75 +TITAN V / V100 | 70 +10X0 / TITAN Xp | 61 +9X0 | 52 +K80 | 37 ## Installation FAQ From 567268854bd7822cd98a438015cb3220f959dc48 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Wed, 1 Feb 2023 12:18:09 -0800 Subject: [PATCH 05/30] Cleanup aabb intersection logic (#1329) --- nerfstudio/cameras/cameras.py | 11 +---- nerfstudio/utils/math.py | 92 +++++++++++++++++++++++------------ 2 files changed, 63 insertions(+), 40 deletions(-) diff --git a/nerfstudio/cameras/cameras.py b/nerfstudio/cameras/cameras.py index 240c771b6..d7d228bae 100644 --- a/nerfstudio/cameras/cameras.py +++ b/nerfstudio/cameras/cameras.py @@ -16,7 +16,6 @@ Camera Models """ import base64 -import importlib import math import os from dataclasses import dataclass @@ -472,15 +471,7 @@ class Cameras(TensorDataclass): rays_o = rays_o.reshape((-1, 3)) rays_d = rays_d.reshape((-1, 3)) - if self._use_nerfacc: - try: - nerfacc = importlib.import_module("nerfacc") - t_min, t_max = nerfacc.ray_aabb_intersect(rays_o, rays_d, tensor_aabb) - except: # pylint: disable=bare-except - t_min, t_max = nerfstudio.utils.math.intersect_aabb(rays_o, rays_d, tensor_aabb) - self._use_nerfacc = False - else: - t_min, t_max = nerfstudio.utils.math.intersect_aabb(rays_o, rays_d, tensor_aabb) + t_min, t_max = nerfstudio.utils.math.intersect_aabb(rays_o, rays_d, tensor_aabb) t_min = t_min.reshape([shape[0], shape[1], 1]) t_max = t_max.reshape([shape[0], shape[1], 1]) diff --git a/nerfstudio/utils/math.py b/nerfstudio/utils/math.py index 2658b1c39..9c2077f4b 100644 --- a/nerfstudio/utils/math.py +++ b/nerfstudio/utils/math.py @@ -15,10 +15,13 @@ """ Math Helper Functions """ from dataclasses import dataclass +from typing import Tuple import torch from torchtyping import TensorType +_USE_NERFACC = True + def components_from_spherical_harmonics(levels: int, directions: TensorType[..., 3]) -> TensorType[..., "components"]: """ @@ -193,47 +196,76 @@ def expected_sin(x_means: torch.Tensor, x_vars: torch.Tensor) -> torch.Tensor: return torch.exp(-0.5 * x_vars) * torch.sin(x_means) -def intersect_aabb(origins, directions, aabb): +@torch.jit.script +def _intersect_aabb( + origins: torch.Tensor, + directions: torch.Tensor, + aabb: torch.Tensor, + max_bound: float = 1e10, + invalid_value: float = 1e10, +) -> Tuple[torch.Tensor, torch.Tensor]: """ - pytorch implementation of ray intersection with AABB box - :param origin: N,3 tensor of 3d positions - :param direction: N,3 tensor of normalized directions - :param aabb: 6,1 array of aabb box in the form of [x_min, y_min, z_min, x_max, y_max, z_max] - :return: t_min, t_max - two tensors of shapes N,1 representing the intersection signed distance from - the origin. t_min is clipped to 0 in case it is negative, in case of no intersection t_min is 1e10. + Implementation of ray intersection with AABB box + + Args: + origins: [N,3] tensor of 3d positions + directions: [N,3] tensor of normalized directions + aabb: [6] array of aabb box in the form of [x_min, y_min, z_min, x_max, y_max, z_max] + max_bound: Maximum value of t_max + invalid_value: Value to return in case of no intersection + + Returns: + t_min, t_max - two tensors of shapes N representing distance of intersection from the origin. """ - max_bound = 1e10 + tx_min = (aabb[:3] - origins) / directions + tx_max = (aabb[3:] - origins) / directions - tx_min = (aabb[0] - origins[:, 0]) / directions[:, 0] - tx_max = (aabb[3] - origins[:, 0]) / directions[:, 0] t_min = torch.min(tx_min, tx_max) t_max = torch.max(tx_min, tx_max) - ty_min_temp = (aabb[1] - origins[:, 1]) / directions[:, 1] - ty_max_temp = (aabb[4] - origins[:, 1]) / directions[:, 1] - ty_min = torch.min(ty_min_temp, ty_max_temp) - ty_max = torch.max(ty_min_temp, ty_max_temp) + t_min = torch.max(t_min, dim=-1).values + t_max = torch.min(t_max, dim=-1).values - cond = torch.logical_or((t_min > ty_max), (ty_min > t_max)) - t_min = torch.where(cond, max_bound, t_min) - t_max = torch.where(cond, max_bound, t_max) + t_min = torch.clamp(t_min, min=0, max=max_bound) + t_max = torch.clamp(t_max, min=0, max=max_bound) - t_min = torch.max(t_min, ty_min) - t_max = torch.min(t_max, ty_max) + cond = t_max <= t_min + t_min = torch.where(cond, invalid_value, t_min) + t_max = torch.where(cond, invalid_value, t_max) - tz_min_temp = (aabb[2] - origins[:, 2]) / directions[:, 2] - tz_max_temp = (aabb[5] - origins[:, 2]) / directions[:, 2] - tz_min = torch.min(tz_min_temp, tz_max_temp) - tz_max = torch.max(tz_min_temp, tz_max_temp) + return t_min, t_max - cond = torch.logical_or((t_min > tz_max), tz_min > t_max) - t_min = torch.where(cond, max_bound, t_min) - t_max = torch.where(cond, max_bound, t_max) - t_min = torch.max(t_min, tz_min) - t_max = torch.min(t_max, tz_max) - t_min = torch.where(t_min > 0, t_min, 0) - t_max = torch.where(t_min == max_bound, max_bound, t_max) +def intersect_aabb( + origins: TensorType["N", 3], + directions: TensorType["N", 3], + aabb: TensorType[6], +) -> Tuple[TensorType["N"], TensorType["N"]]: + """ + Implementation of ray intersection with AABB box + + Args: + origins: 3d positions + directions: Normalized directions + aabb: array of aabb box in the form of [x_min, y_min, z_min, x_max, y_max, z_max] + max_bound: Maximum value of t_max + invalid_value: Value to return in case of no intersection + + Returns: + t_min, t_max - two tensors of shapes N representing distance of intersection from the origin. + """ + + global _USE_NERFACC # pylint: disable=global-statement + if _USE_NERFACC: + try: + import nerfacc # pylint: disable=import-outside-toplevel + + t_min, t_max = nerfacc.ray_aabb_intersect(origins, directions, aabb) + except: # pylint: disable=bare-except + t_min, t_max = _intersect_aabb(origins, directions, aabb, max_bound=1e10, invalid_value=1e10) + _USE_NERFACC = False + else: + t_min, t_max = _intersect_aabb(origins, directions, aabb, max_bound=1e10, invalid_value=1e10) return t_min, t_max From e979f8920093ade6002d7b4848634a2e51f581d4 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Wed, 1 Feb 2023 12:21:57 -0800 Subject: [PATCH 06/30] Avoid protobuf 3.20.0 (#1330) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ca44d209e..21bc69456 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "opencv-python==4.6.0.66", "Pillow>=9.3.0", "plotly>=5.7.0", - "protobuf<=3.20.3", + "protobuf<=3.20.3,!=3.20.0", "pymeshlab>=2022.2.post2", "pyngrok>=5.1.0", "python-socketio>=5.7.1", From 53a830a6917b511c811b09be8641ba0ca7b488a2 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Wed, 1 Feb 2023 13:22:32 -0800 Subject: [PATCH 07/30] Safe normalize (#1332) --- nerfstudio/utils/math.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/nerfstudio/utils/math.py b/nerfstudio/utils/math.py index 9c2077f4b..937866dcc 100644 --- a/nerfstudio/utils/math.py +++ b/nerfstudio/utils/math.py @@ -269,3 +269,19 @@ def intersect_aabb( t_min, t_max = _intersect_aabb(origins, directions, aabb, max_bound=1e10, invalid_value=1e10) return t_min, t_max + + +def safe_normalize( + vectors: TensorType["batch_dim":..., "N"], + eps: float = 1e-10, +) -> TensorType["batch_dim":..., "N"]: + """Normalizes vectors. + + Args: + vectors: Vectors to normalize. + eps: Epsilon value to avoid division by zero. + + Returns: + Normalized vectors. + """ + return vectors / (torch.norm(vectors, dim=-1, keepdim=True) + eps) From 48ec36e53faca773c1bfafc6b9a416456f183f6c Mon Sep 17 00:00:00 2001 From: hturki Date: Wed, 1 Feb 2023 17:55:07 -0500 Subject: [PATCH 08/30] Add support for gradient clipping (#1331) implement optional gradient clipping --- nerfstudio/engine/optimizers.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/nerfstudio/engine/optimizers.py b/nerfstudio/engine/optimizers.py index c97b10580..8281dc6de 100644 --- a/nerfstudio/engine/optimizers.py +++ b/nerfstudio/engine/optimizers.py @@ -18,7 +18,7 @@ Optimizers class. from __future__ import annotations from dataclasses import dataclass -from typing import Any, Dict, List, Type +from typing import Any, Dict, List, Optional, Type import torch from torch.cuda.amp.grad_scaler import GradScaler @@ -36,6 +36,7 @@ class OptimizerConfig(base_config.PrintableConfig): _target: Type = torch.optim.Adam lr: float = 0.0005 eps: float = 1e-08 + max_norm: Optional[float] = None # TODO: somehow make this more generic. i dont like the idea of overriding the setup function # but also not sure how to go about passing things into predefined torch objects. @@ -43,6 +44,7 @@ class OptimizerConfig(base_config.PrintableConfig): """Returns the instantiated object using the config.""" kwargs = vars(self).copy() kwargs.pop("_target") + kwargs.pop("max_norm") return self._target(params, **kwargs) @@ -73,9 +75,11 @@ class Optimizers: self.config = config self.optimizers = {} self.schedulers = {} + self.parameters = {} for param_group_name, params in param_groups.items(): lr_init = config[param_group_name]["optimizer"].lr self.optimizers[param_group_name] = config[param_group_name]["optimizer"].setup(params=params) + self.parameters[param_group_name] = params if config[param_group_name]["scheduler"]: self.schedulers[param_group_name] = config[param_group_name]["scheduler"].setup( optimizer=self.optimizers[param_group_name], lr_init=lr_init @@ -109,13 +113,20 @@ class Optimizers: Args: grad_scaler: GradScaler to use """ - for _, optimizer in self.optimizers.items(): + for param_group, optimizer in self.optimizers.items(): + max_norm = self.config[param_group]["optimizer"].max_norm + if max_norm is not None: + grad_scaler.unscale_(optimizer) + torch.nn.utils.clip_grad_norm_(self.parameters[param_group], max_norm) grad_scaler.step(optimizer) def optimizer_step_all(self): """Run step for all optimizers.""" - for _, optimizer in self.optimizers.items(): + for param_group, optimizer in self.optimizers.items(): # note that they key is the parameter name + max_norm = self.config[param_group]["optimizer"].max_norm + if max_norm is not None: + torch.nn.utils.clip_grad_norm_(self.parameters[param_group], max_norm) optimizer.step() def scheduler_step_all(self, step: int) -> None: From db83068fcd7168d1ac7bdf5bacb7b345bf63cc35 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Wed, 1 Feb 2023 15:15:08 -0800 Subject: [PATCH 09/30] Update optimizer docstrings (#1333) --- nerfstudio/engine/optimizers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nerfstudio/engine/optimizers.py b/nerfstudio/engine/optimizers.py index 8281dc6de..6878eccc2 100644 --- a/nerfstudio/engine/optimizers.py +++ b/nerfstudio/engine/optimizers.py @@ -34,9 +34,13 @@ class OptimizerConfig(base_config.PrintableConfig): """Basic optimizer config with RAdam""" _target: Type = torch.optim.Adam + """The optimizer class to use.""" lr: float = 0.0005 + """The learning rate to use.""" eps: float = 1e-08 + """The epsilon value to use.""" max_norm: Optional[float] = None + """The max norm to use for gradient clipping.""" # TODO: somehow make this more generic. i dont like the idea of overriding the setup function # but also not sure how to go about passing things into predefined torch objects. @@ -54,6 +58,7 @@ class AdamOptimizerConfig(OptimizerConfig): _target: Type = torch.optim.Adam weight_decay: float = 0 + """The weight decay to use.""" @dataclass From 3a416a387bf0a145874826ef32c05ea795c0b290 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Wed, 1 Feb 2023 17:33:06 -0800 Subject: [PATCH 10/30] Dont move viewer outputs to cpu (#1335) --- nerfstudio/viewer/server/viewer_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nerfstudio/viewer/server/viewer_utils.py b/nerfstudio/viewer/server/viewer_utils.py index 44056537f..1a3fefd51 100644 --- a/nerfstudio/viewer/server/viewer_utils.py +++ b/nerfstudio/viewer/server/viewer_utils.py @@ -48,7 +48,6 @@ from nerfstudio.models.base_model import Model from nerfstudio.utils import colormaps, profiler, writer from nerfstudio.utils.decorators import check_main_thread, decorate_all from nerfstudio.utils.io import load_from_json, write_to_json -from nerfstudio.utils.misc import get_dict_to_torch from nerfstudio.utils.writer import GLOBAL_BUFFER, EventName, TimeWriter from nerfstudio.viewer.server.subprocess import run_viewer_bridge_server_as_subprocess from nerfstudio.viewer.server.utils import ( @@ -163,7 +162,6 @@ class RenderThread(threading.Thread): self.exc = e if outputs: - outputs = get_dict_to_torch(outputs) self.vis_outputs = outputs self.state.check_done_render = True From d547279a92216e793e20de959fe352c661be81f2 Mon Sep 17 00:00:00 2001 From: hturki Date: Thu, 2 Feb 2023 12:48:45 -0500 Subject: [PATCH 11/30] add option to log training gradients (#1334) --- nerfstudio/engine/trainer.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index f24ea5ccd..2d4a45395 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -77,6 +77,8 @@ class TrainerConfig(ExperimentConfig): load_step: Optional[int] = None """Optionally specify model step to load from; if none, will find most recent model in load_dir.""" load_config: Optional[Path] = None + """Optionally log gradients during training""" + log_gradients: bool = False class Trainer: @@ -188,7 +190,6 @@ class Trainer: step = 0 for step in range(self._start_step, self._start_step + num_iterations): with TimeWriter(writer, EventName.ITER_TRAIN_TIME, step=step) as train_t: - self.pipeline.train() # training callbacks before the training iteration @@ -370,6 +371,18 @@ class Trainer: loss = functools.reduce(torch.add, loss_dict.values()) self.grad_scaler.scale(loss).backward() # type: ignore self.optimizers.optimizer_scaler_step_all(self.grad_scaler) + + if self.config.log_gradients: + total_grad = 0 + for tag, value in self.pipeline.model.named_parameters(): + assert tag != "Total" + if value.grad is not None: + grad = value.grad.norm() + metrics_dict[f"Gradients/{tag}"] = grad + total_grad += grad + + metrics_dict["Gradients/Total"] = total_grad + self.grad_scaler.update() self.optimizers.scheduler_step_all(step) From ef9e00e6141d2fde83fc344b3e27ac475a8597cd Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Thu, 2 Feb 2023 10:11:41 -0800 Subject: [PATCH 12/30] Check if curl is installed (#1337) * check if curl is installed * lint --- nerfstudio/utils/install_checks.py | 8 ++++++++ scripts/downloads/download_data.py | 3 +++ 2 files changed, 11 insertions(+) diff --git a/nerfstudio/utils/install_checks.py b/nerfstudio/utils/install_checks.py index 23351a1b2..05e505d4e 100644 --- a/nerfstudio/utils/install_checks.py +++ b/nerfstudio/utils/install_checks.py @@ -39,3 +39,11 @@ def check_colmap_installed(): CONSOLE.print("[bold red]Could not find COLMAP. Please install COLMAP.") print("See https://colmap.github.io/install.html for installation instructions.") sys.exit(1) + + +def check_curl_installed(): + """Checks if curl is installed.""" + curl_path = shutil.which("curl") + if curl_path is None: + CONSOLE.print("[bold red]Could not find [yellow]curl[red], Please install [yellow]curl") + sys.exit(1) diff --git a/scripts/downloads/download_data.py b/scripts/downloads/download_data.py index 12a0a0227..89c77391f 100755 --- a/scripts/downloads/download_data.py +++ b/scripts/downloads/download_data.py @@ -15,6 +15,7 @@ from rich.console import Console from typing_extensions import Annotated from nerfstudio.configs.base_config import PrintableConfig +from nerfstudio.utils import install_checks from nerfstudio.utils.scripts import run_command CONSOLE = Console(width=120) @@ -212,6 +213,7 @@ class DNerfDownload(DatasetDownload): """Download the D-NeRF dataset (https://github.com/albertpumarola/D-NeRF).""" # TODO: give this code the same structure as download_nerfstudio + install_checks.check_curl_installed() final_path = save_dir / Path("dnerf") if os.path.exists(final_path): shutil.rmtree(str(final_path)) @@ -253,6 +255,7 @@ class PhototourismDownload(DatasetDownload): def download(self, save_dir: Path): """Download a PhotoTourism dataset: https://www.cs.ubc.ca/~kmyi/imw2020/data.html""" + install_checks.check_curl_installed() if self.capture_name == "all": for capture_name in phototourism_downloads: if capture_name != "all": From 3ab9352be87f408d97eb7569d09382ca38936326 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Thu, 2 Feb 2023 12:55:03 -0800 Subject: [PATCH 13/30] Set min render value for viewer (#1339) --- nerfstudio/viewer/server/viewer_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nerfstudio/viewer/server/viewer_utils.py b/nerfstudio/viewer/server/viewer_utils.py index 1a3fefd51..93dc908b6 100644 --- a/nerfstudio/viewer/server/viewer_utils.py +++ b/nerfstudio/viewer/server/viewer_utils.py @@ -826,7 +826,7 @@ class ViewerState: else: image_height = (num_vis_rays / aspect_ratio) ** 0.5 image_height = int(round(image_height, -1)) - image_height = min(self.max_resolution, image_height) + image_height = max(min(self.max_resolution, image_height), 30) image_width = int(image_height * aspect_ratio) if image_width > self.max_resolution: image_width = self.max_resolution From cf1794acb16b01796f3d1fcb7501df47a6219df0 Mon Sep 17 00:00:00 2001 From: Terrance Wang Date: Thu, 2 Feb 2023 13:07:14 -0800 Subject: [PATCH 14/30] Add lambertian renderer (#1336) * add lambertian shading renderer * adding more functionality * adding to comments * optionally detach normals, fixing naming and typing * Update description --------- Co-authored-by: Matt Tancik Co-authored-by: Matthew Tancik --- nerfstudio/model_components/renderers.py | 34 ++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/nerfstudio/model_components/renderers.py b/nerfstudio/model_components/renderers.py index 24ce46085..1f95de95f 100644 --- a/nerfstudio/model_components/renderers.py +++ b/nerfstudio/model_components/renderers.py @@ -332,3 +332,37 @@ class NormalsRenderer(nn.Module): """Calculate normals along the ray.""" n = torch.sum(weights * normals, dim=-2) return n + + +class LambertianShadingRenderer(nn.Module): + """Calculate Lambertian shading.""" + + @classmethod + def forward( + cls, + rgb: TensorType["bs":..., 3], + normals: TensorType["bs":..., 3], + light_direction: TensorType["bs":..., 3], + shading_weight: float = 1.0, + detach_normals=True, + ): + """Calculate Lambertian shading. + + Args: + rgb: Accumulated rgb along a ray. + normals: Accumulated normals along a ray. + light_direction: Direction of light source. + shading_weight: Lambertian shading (1.0) vs. ambient lighting (0.0) ratio + detach_normals: Detach normals from the computation graph when computing shading. + + Returns: + Textureless Lambertian shading, Lambertian shading + """ + if detach_normals: + normals = normals.detach() + + lambertian = (1 - shading_weight) + shading_weight * (normals @ light_direction).clamp(min=0) + shaded = lambertian.unsqueeze(-1).repeat(1, 3) + shaded_albedo = rgb * lambertian.unsqueeze(-1) + + return shaded, shaded_albedo From 2e79c93cd2562e8bb8c97abf5104cd13141ee804 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Thu, 2 Feb 2023 18:06:11 -0800 Subject: [PATCH 15/30] Fix normals visualization (#1341) --- nerfstudio/model_components/renderers.py | 13 +++++++++++-- nerfstudio/models/nerfacto.py | 6 ++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/nerfstudio/model_components/renderers.py b/nerfstudio/model_components/renderers.py index 1f95de95f..6a8e2a5f9 100644 --- a/nerfstudio/model_components/renderers.py +++ b/nerfstudio/model_components/renderers.py @@ -37,7 +37,7 @@ from torchtyping import TensorType from typing_extensions import Literal from nerfstudio.cameras.rays import RaySamples -from nerfstudio.utils.math import components_from_spherical_harmonics +from nerfstudio.utils.math import components_from_spherical_harmonics, safe_normalize BACKGROUND_COLOR_OVERRIDE: Optional[TensorType[3]] = None @@ -328,9 +328,18 @@ class NormalsRenderer(nn.Module): cls, normals: TensorType["bs":..., "num_samples", 3], weights: TensorType["bs":..., "num_samples", 1], + normalize: bool = True, ) -> TensorType["bs":..., 3]: - """Calculate normals along the ray.""" + """Calculate normals along the ray. + + Args: + normals: Normals for each sample. + weights: Weights of each sample. + normalize: Normalize normals. + """ n = torch.sum(weights * normals, dim=-2) + if normalize: + n = safe_normalize(n) return n diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py index c996e20b3..91b044884 100644 --- a/nerfstudio/models/nerfacto.py +++ b/nerfstudio/models/nerfacto.py @@ -250,8 +250,10 @@ class NerfactoModel(Model): } if self.config.predict_normals: - outputs["normals"] = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights) - outputs["pred_normals"] = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights) + normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights) + pred_normals = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights) + outputs["normals"] = (normals + 1) / 2 + outputs["pred_normals"] = (pred_normals + 1) / 2 # These use a lot of GPU memory, so we avoid storing them for eval. if self.training: From a0976d1ff82a4cd3f34860071fefaa541b589357 Mon Sep 17 00:00:00 2001 From: Jeffrey Hu <102542936+jeffreyhuparallel@users.noreply.github.com> Date: Thu, 2 Feb 2023 18:22:55 -0800 Subject: [PATCH 16/30] Fix depth-supervised nerf when using z-buffer depth. (#1340) * fix z buffer depth * remove script --- nerfstudio/model_components/losses.py | 2 +- nerfstudio/models/depth_nerfacto.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nerfstudio/model_components/losses.py b/nerfstudio/model_components/losses.py index 7fcfe0419..6a981d664 100644 --- a/nerfstudio/model_components/losses.py +++ b/nerfstudio/model_components/losses.py @@ -291,7 +291,7 @@ def depth_loss( Depth loss scalar. """ if not is_euclidean: - termination_depth = termination_depth / directions_norm + termination_depth = termination_depth * directions_norm steps = (ray_samples.frustums.starts + ray_samples.frustums.ends) / 2 if depth_loss_type == DephtLossType.DS_NERF: diff --git a/nerfstudio/models/depth_nerfacto.py b/nerfstudio/models/depth_nerfacto.py index da4c59002..ac0768fdf 100644 --- a/nerfstudio/models/depth_nerfacto.py +++ b/nerfstudio/models/depth_nerfacto.py @@ -108,7 +108,7 @@ class DepthNerfactoModel(NerfactoModel): metrics, images = super().get_image_metrics_and_images(outputs, batch) ground_truth_depth = batch["depth_image"] if not self.config.is_euclidean_depth: - ground_truth_depth = ground_truth_depth / outputs["directions_norm"] + ground_truth_depth = ground_truth_depth * outputs["directions_norm"] ground_truth_depth_colormap = colormaps.apply_depth_colormap(ground_truth_depth) predicted_depth_colormap = colormaps.apply_depth_colormap( From 59d7e99c804de7588b9ffd5d993f1c667a2a5d23 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Fri, 3 Feb 2023 11:21:54 -0800 Subject: [PATCH 17/30] Create shaders (#1344) --- nerfstudio/model_components/renderers.py | 34 ----------- nerfstudio/model_components/shaders.py | 77 ++++++++++++++++++++++++ nerfstudio/models/nerfacto.py | 15 ++--- 3 files changed, 83 insertions(+), 43 deletions(-) create mode 100644 nerfstudio/model_components/shaders.py diff --git a/nerfstudio/model_components/renderers.py b/nerfstudio/model_components/renderers.py index 6a8e2a5f9..d13c29233 100644 --- a/nerfstudio/model_components/renderers.py +++ b/nerfstudio/model_components/renderers.py @@ -341,37 +341,3 @@ class NormalsRenderer(nn.Module): if normalize: n = safe_normalize(n) return n - - -class LambertianShadingRenderer(nn.Module): - """Calculate Lambertian shading.""" - - @classmethod - def forward( - cls, - rgb: TensorType["bs":..., 3], - normals: TensorType["bs":..., 3], - light_direction: TensorType["bs":..., 3], - shading_weight: float = 1.0, - detach_normals=True, - ): - """Calculate Lambertian shading. - - Args: - rgb: Accumulated rgb along a ray. - normals: Accumulated normals along a ray. - light_direction: Direction of light source. - shading_weight: Lambertian shading (1.0) vs. ambient lighting (0.0) ratio - detach_normals: Detach normals from the computation graph when computing shading. - - Returns: - Textureless Lambertian shading, Lambertian shading - """ - if detach_normals: - normals = normals.detach() - - lambertian = (1 - shading_weight) + shading_weight * (normals @ light_direction).clamp(min=0) - shaded = lambertian.unsqueeze(-1).repeat(1, 3) - shaded_albedo = rgb * lambertian.unsqueeze(-1) - - return shaded, shaded_albedo diff --git a/nerfstudio/model_components/shaders.py b/nerfstudio/model_components/shaders.py new file mode 100644 index 000000000..e71388e27 --- /dev/null +++ b/nerfstudio/model_components/shaders.py @@ -0,0 +1,77 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shaders for rendering.""" +from typing import Optional + +from torch import nn +from torchtyping import TensorType + + +class LambertianShader(nn.Module): + """Calculate Lambertian shading.""" + + @classmethod + def forward( + cls, + rgb: TensorType["bs":..., 3], + normals: TensorType["bs":..., 3], + light_direction: TensorType["bs":..., 3], + shading_weight: float = 1.0, + detach_normals=True, + ): + """Calculate Lambertian shading. + + Args: + rgb: Accumulated rgb along a ray. + normals: Accumulated normals along a ray. + light_direction: Direction of light source. + shading_weight: Lambertian shading (1.0) vs. ambient lighting (0.0) ratio + detach_normals: Detach normals from the computation graph when computing shading. + + Returns: + Textureless Lambertian shading, Lambertian shading + """ + if detach_normals: + normals = normals.detach() + + lambertian = (1 - shading_weight) + shading_weight * (normals @ light_direction).clamp(min=0) + shaded = lambertian.unsqueeze(-1).repeat(1, 3) + shaded_albedo = rgb * lambertian.unsqueeze(-1) + + return shaded, shaded_albedo + + +class NormalsShader(nn.Module): + """Calculate shading for normals.""" + + @classmethod + def forward( + cls, + normals: TensorType["bs":..., 3], + weights: Optional[TensorType["bs":..., 1]] = None, + ): + """Applies a rainbow colormap to the normals. + + Args: + normals: Normalized 3D vectors. + weights: Optional weights to scale to the normal colors. (Can be used for masking) + + Returns: + Colored normals + """ + normals = (normals + 1) / 2 + if weights is not None: + normals = normals * weights + return normals diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py index 91b044884..7442bfe8b 100644 --- a/nerfstudio/models/nerfacto.py +++ b/nerfstudio/models/nerfacto.py @@ -54,6 +54,7 @@ from nerfstudio.model_components.renderers import ( RGBRenderer, ) from nerfstudio.model_components.scene_colliders import NearFarCollider +from nerfstudio.model_components.shaders import NormalsShader from nerfstudio.models.base_model import Model, ModelConfig from nerfstudio.utils import colormaps @@ -187,6 +188,9 @@ class NerfactoModel(Model): self.renderer_depth = DepthRenderer() self.renderer_normals = NormalsRenderer() + # shaders + self.normals_shader = NormalsShader() + # losses self.rgb_loss = MSELoss() @@ -252,9 +256,8 @@ class NerfactoModel(Model): if self.config.predict_normals: normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights) pred_normals = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights) - outputs["normals"] = (normals + 1) / 2 - outputs["pred_normals"] = (pred_normals + 1) / 2 - + outputs["normals"] = self.normals_shader(normals) + outputs["pred_normals"] = self.normals_shader(pred_normals) # These use a lot of GPU memory, so we avoid storing them for eval. if self.training: outputs["weights_list"] = weights_list @@ -336,12 +339,6 @@ class NerfactoModel(Model): images_dict = {"img": combined_rgb, "accumulation": combined_acc, "depth": combined_depth} - # normals to RGB for visualization. TODO: use a colormap - if "normals" in outputs: - images_dict["normals"] = (outputs["normals"] + 1.0) / 2.0 - if "pred_normals" in outputs: - images_dict["pred_normals"] = (outputs["pred_normals"] + 1.0) / 2.0 - for i in range(self.config.num_proposal_iterations): key = f"prop_depth_{i}" prop_depth_i = colormaps.apply_depth_colormap( From 319a2b965c4a578a413fb81ad6f90e6ac4fae4c5 Mon Sep 17 00:00:00 2001 From: machenmusik Date: Fri, 3 Feb 2023 16:38:40 -0500 Subject: [PATCH 18/30] New config parameter --viewer.codec defaults to VP8, but allows H264 (#1343) * New config parameter --viewer.codec defaults to H264 rather than VP8 * make quantize clearer * change codec to Literal default is VP8 due to libx264 log spam, see https://github.com/nerfstudio-project/nerfstudio/pull/1343#issuecomment-1416235500 * isort * pylint, including python 3.7 compatibility --- nerfstudio/configs/base_config.py | 4 ++++ nerfstudio/viewer/server/utils.py | 5 +++-- nerfstudio/viewer/server/viewer_utils.py | 10 ++++++++-- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/nerfstudio/configs/base_config.py b/nerfstudio/configs/base_config.py index 548324069..a85b5ddaa 100644 --- a/nerfstudio/configs/base_config.py +++ b/nerfstudio/configs/base_config.py @@ -23,6 +23,8 @@ from dataclasses import dataclass from pathlib import Path from typing import Any, List, Optional, Tuple, Type +from typing_extensions import Literal + # model instances from nerfstudio.utils import writer @@ -149,3 +151,5 @@ class ViewerConfig(PrintableConfig): skip_openrelay: bool = False """Avoid using openrelay to communicate with the viewer. Try disabling if you have trouble connecting to the viewer""" + codec: Literal["H264", "VP8"] = "VP8" + """Video codec that viewer will use.""" diff --git a/nerfstudio/viewer/server/utils.py b/nerfstudio/viewer/server/utils.py index b0763b6d1..a21b02d82 100644 --- a/nerfstudio/viewer/server/utils.py +++ b/nerfstudio/viewer/server/utils.py @@ -63,7 +63,7 @@ def three_js_perspective_camera_focal_length(fov: float, image_height: int): def get_intrinsics_matrix_and_camera_to_world_h( - camera_object: Dict[str, Any], image_height: int + camera_object: Dict[str, Any], image_height: int, image_width: Optional[int] = None ) -> Tuple[torch.Tensor, torch.Tensor]: """Returns the camera intrinsics matrix and the camera to world homogeneous matrix. @@ -74,7 +74,8 @@ def get_intrinsics_matrix_and_camera_to_world_h( # intrinsics fov = camera_object["fov"] aspect = camera_object["aspect"] - image_width = aspect * image_height + if image_width is None: + image_width = aspect * image_height pp_w = image_width / 2.0 pp_h = image_height / 2.0 if (camera_object["camera_type"] == "perspective") | (camera_object["camera_type"] == "fisheye"): diff --git a/nerfstudio/viewer/server/viewer_utils.py b/nerfstudio/viewer/server/viewer_utils.py index 93dc908b6..b59381f26 100644 --- a/nerfstudio/viewer/server/viewer_utils.py +++ b/nerfstudio/viewer/server/viewer_utils.py @@ -689,7 +689,8 @@ class ViewerState: video = SingleFrameStreamTrack() self.video_tracks.add(video) video_sender = pc.addTrack(video) - force_codec(pc, video_sender, "video/VP8") + print(f"viewer using video/{self.config.codec}") + force_codec(pc, video_sender, f"video/{self.config.codec}") await pc.setRemoteDescription(offer) answer = await pc.createAnswer() @@ -831,6 +832,11 @@ class ViewerState: if image_width > self.max_resolution: image_width = self.max_resolution image_height = int(image_width / aspect_ratio) + if self.config.codec != "VP8": + # force even values to allow hardware encoder usage + quantize = 2 + image_width = int(image_width / quantize) * quantize + image_height = int(image_height / quantize) * quantize return image_height, image_width def _process_invalid_output(self, output_type: str) -> str: @@ -904,7 +910,7 @@ class ViewerState: return intrinsics_matrix, camera_to_world_h = get_intrinsics_matrix_and_camera_to_world_h( - camera_object, image_height=image_height + camera_object, image_height=image_height, image_width=image_width ) camera_to_world = camera_to_world_h[:3, :] From fad5a839aac0e51e326173d65144a726b358b514 Mon Sep 17 00:00:00 2001 From: machenmusik Date: Fri, 3 Feb 2023 21:01:37 -0500 Subject: [PATCH 19/30] double-quote filenames in process-data commands so that on Windows, paths with spaces don't fail (#1347) * double-quote filenames in commands so that on Windows, paths with spaces don't fail * reformatting by black --- nerfstudio/process_data/process_data_utils.py | 18 +++++++++--------- scripts/process_data.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/nerfstudio/process_data/process_data_utils.py b/nerfstudio/process_data/process_data_utils.py index d8d9206e9..cedfc2b27 100644 --- a/nerfstudio/process_data/process_data_utils.py +++ b/nerfstudio/process_data/process_data_utils.py @@ -78,8 +78,8 @@ def get_num_frames_in_video(video: Path) -> int: Returns: The number of frames in a video. """ - cmd = f"ffprobe -v error -select_streams v:0 -count_packets \ - -show_entries stream=nb_read_packets -of csv=p=0 {video}" + cmd = f'ffprobe -v error -select_streams v:0 -count_packets \ + -show_entries stream=nb_read_packets -of csv=p=0 "{video}"' output = run_command(cmd) assert output is not None output = output.strip(" ,\t\n\r") @@ -114,7 +114,7 @@ def convert_video_to_images( print("Number of frames in video:", num_frames) out_filename = image_dir / "frame_%05d.png" - ffmpeg_cmd = f"ffmpeg -i {video_path}" + ffmpeg_cmd = f'ffmpeg -i "{video_path}"' spacing = num_frames // num_frames_target if spacing > 1: @@ -168,7 +168,7 @@ def copy_images_list( file_type = image_paths[0].suffix filename = f"frame_%05d{file_type}" crop = f"crop=iw-{crop_border_pixels*2}:ih-{crop_border_pixels*2}" - ffmpeg_cmd = f"ffmpeg -y -noautorotate -i {image_dir / filename} -q:v 2 -vf {crop} {image_dir / filename}" + ffmpeg_cmd = f'ffmpeg -y -noautorotate -i "{image_dir / filename}" -q:v 2 -vf {crop} "{image_dir / filename}"' run_command(ffmpeg_cmd, verbose=verbose) num_frames = len(image_paths) @@ -210,9 +210,9 @@ def copy_and_upscale_polycam_depth_maps_list( for idx, depth_map in enumerate(polycam_depth_image_filenames): destination = depth_dir / f"frame_{idx + 1:05d}{depth_map.suffix}" ffmpeg_cmd = [ - f"ffmpeg -y -i {depth_map} ", + f'ffmpeg -y -i "{depth_map}" ', f"-q:v 2 -vf scale=iw*{upscale_factor}:ih*{upscale_factor}:flags=neighbor ", - f"{destination}", + f'"{destination}"', ] ffmpeg_cmd = " ".join(ffmpeg_cmd) run_command(ffmpeg_cmd, verbose=verbose) @@ -222,7 +222,7 @@ def copy_and_upscale_polycam_depth_maps_list( file_type = depth_dir.glob("frame_*").__next__().suffix filename = f"frame_%05d{file_type}" crop = f"crop=iw-{crop_border_pixels * 2}:ih-{crop_border_pixels * 2}" - ffmpeg_cmd = f"ffmpeg -y -i {depth_dir / filename} -q:v 2 -vf {crop} {depth_dir / filename}" + ffmpeg_cmd = f'ffmpeg -y -i "{depth_dir / filename}" -q:v 2 -vf {crop} "{depth_dir / filename}"' run_command(ffmpeg_cmd, verbose=verbose) CONSOLE.log("[bold green]:tada: Done upscaling depth maps.") @@ -282,9 +282,9 @@ def downscale_images(image_dir: Path, num_downscales: int, folder_name: str = "i for f in files: filename = f.name ffmpeg_cmd = [ - f"ffmpeg -y -noautorotate -i {image_dir / filename} ", + f'ffmpeg -y -noautorotate -i "{image_dir / filename}" ', f"-q:v 2 -vf scale=iw/{downscale_factor}:ih/{downscale_factor} ", - f"{downscale_dir / filename}", + f'"{downscale_dir / filename}"', ] ffmpeg_cmd = " ".join(ffmpeg_cmd) run_command(ffmpeg_cmd, verbose=verbose) diff --git a/scripts/process_data.py b/scripts/process_data.py index 4727e2b2c..4d8e9acf7 100755 --- a/scripts/process_data.py +++ b/scripts/process_data.py @@ -350,7 +350,7 @@ class ProcessInsta360: if not filename_back.exists(): raise FileNotFoundError(f"Could not find {filename_back}") - ffprobe_cmd = f"ffprobe -v quiet -print_format json -show_streams -select_streams v:0 {filename_back}" + ffprobe_cmd = f'ffprobe -v quiet -print_format json -show_streams -select_streams v:0 "{filename_back}"' ffprobe_output = process_data_utils.run_command(ffprobe_cmd) From 9f5da513192568a34520652d5253bc8c26bae3d4 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Fri, 3 Feb 2023 23:15:05 -0800 Subject: [PATCH 20/30] Improve viewer colormap choices (#1348) --- nerfstudio/viewer/app/package.json | 2 +- .../src/modules/ConfigPanel/ConfigPanel.jsx | 51 +++++++++ nerfstudio/viewer/app/src/reducer.js | 9 +- nerfstudio/viewer/server/viewer_utils.py | 103 ++++++++++-------- 4 files changed, 116 insertions(+), 49 deletions(-) diff --git a/nerfstudio/viewer/app/package.json b/nerfstudio/viewer/app/package.json index 37f2b7bbb..171ec8dd1 100644 --- a/nerfstudio/viewer/app/package.json +++ b/nerfstudio/viewer/app/package.json @@ -1,7 +1,7 @@ { "name": "viewer", "homepage": ".", - "version": "23-01-25-0", + "version": "23-02-3-0", "private": true, "dependencies": { "@emotion/react": "^11.10.4", diff --git a/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx b/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx index a1c331fad..66b92458e 100644 --- a/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx @@ -37,6 +37,12 @@ export function RenderControls() { const colormapChoice = useSelector( (state) => state.renderingState.colormap_choice, ); + const colormapInvert = useSelector( + (state) => state.renderingState.colormap_invert, + ); + const colormapNormalize = useSelector( + (state) => state.renderingState.colormap_normalize, + ); const max_resolution = useSelector( (state) => state.renderingState.maxResolution, ); @@ -115,6 +121,51 @@ export function RenderControls() { }, disabled: colormapOptions.length === 1, }, + colormap_invert: { + label: '| Invert', + value: colormapInvert, + hint: 'Invert the colormap', + onChange: (v) => { + dispatch_and_send( + websocket, + dispatch, + 'renderingState/colormap_invert', + v, + ); + }, + render: (get) => get('colormap_options') !== 'default', + }, + colormap_normalize: { + label: '| Normalize', + value: colormapNormalize, + hint: 'Whether to normalize output between 0 and 1', + onChange: (v) => { + dispatch_and_send( + websocket, + dispatch, + 'renderingState/colormap_normalize', + v, + ); + }, + render: (get) => get('colormap_options') !== 'default', + }, + colormap_range: { + label: '| Range', + value: [0, 1], + step: 0.01, + min: -2, + max: 5, + hint: 'Min and max values of the colormap', + onChange: (v) => { + dispatch_and_send( + websocket, + dispatch, + 'renderingState/colormap_range', + v, + ); + }, + render: (get) => get('colormap_options') !== 'default', + }, // Dynamic Resolution target_train_util: { label: 'Train Util.', diff --git a/nerfstudio/viewer/app/src/reducer.js b/nerfstudio/viewer/app/src/reducer.js index 6de1ea7c2..d8335c80d 100644 --- a/nerfstudio/viewer/app/src/reducer.js +++ b/nerfstudio/viewer/app/src/reducer.js @@ -30,12 +30,17 @@ const initialState = { all_camera_paths: null, // object containing camera paths and names - isTraining: true, + + // colormap options output_options: ['rgb'], // populated by the possible Graph outputs output_choice: 'rgb', // the selected output colormap_options: ['default'], // populated by the output choice colormap_choice: 'default', // the selected colormap + colormap_invert: false, // whether to invert the colormap + colormap_normalize: false, // whether to normalize the colormap + colormap_range: [0.0, 1.0], // the range of the colormap + maxResolution: 1024, targetTrainUtil: 0.9, eval_res: '?', @@ -51,7 +56,7 @@ const initialState = { // Crop Box Options crop_enabled: false, - crop_bg_color: {r: 38, g:42, b:55}, + crop_bg_color: { r: 38, g: 42, b: 55 }, crop_scale: [2.0, 2.0, 2.0], crop_center: [0.0, 0.0, 0.0], }, diff --git a/nerfstudio/viewer/server/viewer_utils.py b/nerfstudio/viewer/server/viewer_utils.py index b59381f26..ce0858bb4 100644 --- a/nerfstudio/viewer/server/viewer_utils.py +++ b/nerfstudio/viewer/server/viewer_utils.py @@ -82,7 +82,7 @@ def setup_viewer(config: cfg.ViewerConfig, log_filename: Path, datapath: str): class OutputTypes(str, enum.Enum): - """Noncomprehsnive list of output render types""" + """Noncomprehensive list of output render types""" INIT = "init" RGB = "rgb" @@ -92,14 +92,14 @@ class OutputTypes(str, enum.Enum): class ColormapTypes(str, enum.Enum): - """Noncomprehsnive list of colormap render types""" + """List of colormap render types""" - INIT = "init" DEFAULT = "default" TURBO = "turbo" - DEPTH = "depth" - SEMANTIC = "semantic" - BOOLEAN = "boolean" + VIRIDIS = "viridis" + MAGMA = "magma" + INFERNO = "inferno" + CIVIDIS = "cividis" class IOChangeException(Exception): @@ -220,12 +220,15 @@ class CheckThread(threading.Thread): # check colormap type colormap_type = self.state.vis["renderingState/colormap_choice"].read() - if colormap_type is None: - colormap_type = ColormapTypes.INIT if self.state.prev_colormap_type != colormap_type: self.state.check_interrupt_vis = True return + colormap_range = self.state.vis["renderingState/colormap_range"].read() + if self.state.prev_colormap_range != colormap_range: + self.state.check_interrupt_vis = True + return + # check max render max_resolution = self.state.vis["renderingState/maxResolution"].read() if max_resolution is not None: @@ -285,7 +288,10 @@ class ViewerState: self.prev_camera_matrix = None self.prev_render_time = 0 self.prev_output_type = OutputTypes.INIT - self.prev_colormap_type = ColormapTypes.INIT + self.prev_colormap_type = None + self.prev_colormap_invert = False + self.prev_colormap_normalize = False + self.prev_colormap_range = [0, 1] self.prev_moving = False self.output_type_changed = True self.max_resolution = 1000 @@ -583,11 +589,21 @@ class ViewerState: self.camera_moving = True colormap_type = self.vis["renderingState/colormap_choice"].read() - if colormap_type is None: - colormap_type = ColormapTypes.INIT if self.prev_colormap_type != colormap_type: self.camera_moving = True + colormap_range = self.vis["renderingState/colormap_range"].read() + if self.prev_colormap_range != colormap_range: + self.camera_moving = True + + colormap_invert = self.vis["renderingState/colormap_invert"].read() + if self.prev_colormap_invert != colormap_invert: + self.camera_moving = True + + colormap_normalize = self.vis["renderingState/colormap_normalize"].read() + if self.prev_colormap_normalize != colormap_normalize: + self.camera_moving = True + crop_bg_color = self.vis["renderingState/crop_bg_color"].read() if self.prev_crop_enabled: if self.prev_crop_bg_color != crop_bg_color: @@ -621,37 +637,28 @@ class ViewerState: return outputs[reformatted_output] # rendering depth outputs - if self.prev_colormap_type == ColormapTypes.DEPTH or ( - self.prev_colormap_type == ColormapTypes.DEFAULT - and outputs[reformatted_output].dtype == torch.float - and (torch.max(outputs[reformatted_output]) - 1.0) > eps # handle floating point arithmetic - ): - accumulation_str = ( - OutputTypes.ACCUMULATION - if OutputTypes.ACCUMULATION in self.output_list - else OutputTypes.ACCUMULATION_FINE - ) - return colormaps.apply_depth_colormap(outputs[reformatted_output], accumulation=outputs[accumulation_str]) - - # rendering accumulation outputs - if self.prev_colormap_type == ColormapTypes.TURBO or ( - self.prev_colormap_type == ColormapTypes.DEFAULT and outputs[reformatted_output].dtype == torch.float - ): - return colormaps.apply_colormap(outputs[reformatted_output]) + if outputs[reformatted_output].shape[-1] == 1 and outputs[reformatted_output].dtype == torch.float: + output = outputs[reformatted_output] + if self.prev_colormap_normalize: + output = output - torch.min(output) + output = output / (torch.max(output) + eps) + output = output * (self.prev_colormap_range[1] - self.prev_colormap_range[0]) + self.prev_colormap_range[0] + output = torch.clip(output, 0, 1) + if self.prev_colormap_invert: + output = 1 - output + if self.prev_colormap_type == ColormapTypes.DEFAULT: + return colormaps.apply_colormap(output, cmap=ColormapTypes.TURBO.value) + return colormaps.apply_colormap(output, cmap=self.prev_colormap_type) # rendering semantic outputs - if self.prev_colormap_type == ColormapTypes.SEMANTIC or ( - self.prev_colormap_type == ColormapTypes.DEFAULT and outputs[reformatted_output].dtype == torch.int - ): + if outputs[reformatted_output].dtype == torch.int: logits = outputs[reformatted_output] labels = torch.argmax(torch.nn.functional.softmax(logits, dim=-1), dim=-1) # type: ignore assert colors is not None return colors[labels] # rendering boolean outputs - if self.prev_colormap_type == ColormapTypes.BOOLEAN or ( - self.prev_colormap_type == ColormapTypes.DEFAULT and outputs[reformatted_output].dtype == torch.bool - ): + if outputs[reformatted_output].dtype == torch.bool: return colormaps.apply_boolean_colormap(outputs[reformatted_output]) raise NotImplementedError @@ -713,13 +720,12 @@ class ViewerState: for video_track in self.video_tracks: video_track.put_frame(image) - def _send_output_to_viewer(self, outputs: Dict[str, Any], colors: torch.Tensor = None, eps=1e-6): + def _send_output_to_viewer(self, outputs: Dict[str, Any], colors: torch.Tensor = None): """Chooses the correct output and sends it to the viewer Args: outputs: the dictionary of outputs to choose from, from the graph colors: is only set if colormap is for semantics. Defaults to None. - eps: epsilon to handle floating point comparisons """ if self.output_list is None: self.output_list = list(outputs.keys()) @@ -732,16 +738,13 @@ class ViewerState: reformatted_output = self._process_invalid_output(self.prev_output_type) # re-register colormaps and send to viewer - if self.output_type_changed or self.prev_colormap_type == ColormapTypes.INIT: + if self.output_type_changed or self.prev_colormap_type is None: self.prev_colormap_type = ColormapTypes.DEFAULT - colormap_options = [ColormapTypes.DEFAULT] - if ( - outputs[reformatted_output].shape[-1] != 3 - and outputs[reformatted_output].dtype == torch.float - and (torch.max(outputs[reformatted_output]) - 1.0) <= eps # handle floating point arithmetic - ): - # accumulation can also include depth - colormap_options.extend(["depth"]) + colormap_options = [] + if outputs[reformatted_output].shape[-1] == 3: + colormap_options = [ColormapTypes.DEFAULT] + if outputs[reformatted_output].shape[-1] == 1 and outputs[reformatted_output].dtype == torch.float: + colormap_options = list(ColormapTypes) self.output_type_changed = False self.vis["renderingState/colormap_choice"].write(self.prev_colormap_type) self.vis["renderingState/colormap_options"].write(colormap_options) @@ -885,9 +888,17 @@ class ViewerState: # check and perform colormap type updates colormap_type = self.vis["renderingState/colormap_choice"].read() - colormap_type = ColormapTypes.INIT if colormap_type is None else colormap_type self.prev_colormap_type = colormap_type + colormap_invert = self.vis["renderingState/colormap_invert"].read() + self.prev_colormap_invert = colormap_invert + + colormap_normalize = self.vis["renderingState/colormap_normalize"].read() + self.prev_colormap_normalize = colormap_normalize + + colormap_range = self.vis["renderingState/colormap_range"].read() + self.prev_colormap_range = colormap_range + # update render aabb try: self._update_render_aabb(graph) From cddf6a6569ac5f5c84a33ce2a856b2c1fd57de14 Mon Sep 17 00:00:00 2001 From: ManuConcepBrito <33829944+ManuConcepBrito@users.noreply.github.com> Date: Sun, 5 Feb 2023 20:04:53 +0100 Subject: [PATCH 21/30] Consistent naming for depth maps in Polycam (#1352) --- nerfstudio/process_data/polycam_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nerfstudio/process_data/polycam_utils.py b/nerfstudio/process_data/polycam_utils.py index d1887833b..079ff16f7 100644 --- a/nerfstudio/process_data/polycam_utils.py +++ b/nerfstudio/process_data/polycam_utils.py @@ -72,7 +72,7 @@ def polycam_to_json( frame["h"] = frame_json["height"] - crop_border_pixels * 2 frame["file_path"] = f"./images/frame_{i+1:05d}{image_filename.suffix}" if use_depth: - frame["depth_map_path"] = f"./depth/frame_{i+1:05d}{depth_filenames[i].suffix}" + frame["depth_file_path"] = f"./depth/frame_{i+1:05d}{depth_filenames[i].suffix}" # Transform matrix to nerfstudio format. Please refer to the documentation for coordinate system conventions. frame["transform_matrix"] = [ [frame_json["t_20"], frame_json["t_21"], frame_json["t_22"], frame_json["t_23"]], From 2ffe4f903605c1d84dc2a45601360c3a40f45be0 Mon Sep 17 00:00:00 2001 From: "Francisca T. Gil Ureta" <124404718+gilureta@users.noreply.github.com> Date: Mon, 6 Feb 2023 13:22:19 -0500 Subject: [PATCH 22/30] Handle uncalibrated Metashape sensor (#1359) This fix allows loading Metashape XML files which contain an additional uncalibrated sensor. Metashape XML format keeps unmatched images but assignes a dummy sensor to them. The sensor has no calibration data. Co-authored-by: gilureta --- nerfstudio/process_data/metashape_utils.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/nerfstudio/process_data/metashape_utils.py b/nerfstudio/process_data/metashape_utils.py index 491c06a2c..17d7771e8 100644 --- a/nerfstudio/process_data/metashape_utils.py +++ b/nerfstudio/process_data/metashape_utils.py @@ -58,10 +58,14 @@ def metashape_to_json( # pylint: disable=too-many-statements sensors = chunk.find("sensors") # TODO Add support for per-frame intrinsics - if sensors is None or len(sensors) != 1: + if sensors is None: + raise ValueError("No sensors found") + + calibrated_sensors = [sensor for sensor in sensors if sensor.find("calibration")] + if len(calibrated_sensors) != 1: raise ValueError("Only one sensor is supported for now") - sensor = sensors.find("sensor") + sensor = calibrated_sensors[0] data = {} @@ -98,6 +102,14 @@ def metashape_to_json( # pylint: disable=too-many-statements if camera_label not in image_filename_map: continue frame["file_path"] = image_filename_map[camera_label].as_posix() + + if camera.get("sensor_id") != sensor.get("id"): + # this should only happen when we have a sensor that doesn't have calibration + if verbose: + CONSOLE.print(f"Missing sensor calibration for {camera.get('label')}, Skipping") + num_skipped += 1 + continue + if camera.find("transform") is None: if verbose: CONSOLE.print(f"Missing transforms data for {camera.get('label')}, Skipping") From bf6476912878f61912483111c6c01e622d3d7a12 Mon Sep 17 00:00:00 2001 From: machenmusik Date: Mon, 6 Feb 2023 15:53:27 -0500 Subject: [PATCH 23/30] add viewer.local config property when viewing local instance (#1358) If running local server instance, avoid using relays to communicate with the viewer. --- nerfstudio/configs/base_config.py | 2 ++ nerfstudio/viewer/server/viewer_utils.py | 43 +++++++++++++----------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/nerfstudio/configs/base_config.py b/nerfstudio/configs/base_config.py index a85b5ddaa..a2e695c14 100644 --- a/nerfstudio/configs/base_config.py +++ b/nerfstudio/configs/base_config.py @@ -153,3 +153,5 @@ class ViewerConfig(PrintableConfig): connecting to the viewer""" codec: Literal["H264", "VP8"] = "VP8" """Video codec that viewer will use.""" + local: bool = False + """If running local server instance, avoid using relays to communicate with the viewer.""" diff --git a/nerfstudio/viewer/server/viewer_utils.py b/nerfstudio/viewer/server/viewer_utils.py index ce0858bb4..2c1541467 100644 --- a/nerfstudio/viewer/server/viewer_utils.py +++ b/nerfstudio/viewer/server/viewer_utils.py @@ -669,28 +669,31 @@ class ViewerState: # returns the description to for WebRTC to the specific websocket connection offer = RTCSessionDescription(data["sdp"], data["type"]) - if self.config.skip_openrelay: - ice_servers = [ - RTCIceServer(urls="stun:stun.l.google.com:19302"), - ] + if self.config.local: + pc = RTCPeerConnection() else: - ice_servers = [ - RTCIceServer(urls="stun:stun.l.google.com:19302"), - RTCIceServer(urls="stun:openrelay.metered.ca:80"), - RTCIceServer( - urls="turn:openrelay.metered.ca:80", username="openrelayproject", credential="openrelayproject" - ), - RTCIceServer( - urls="turn:openrelay.metered.ca:443", username="openrelayproject", credential="openrelayproject" - ), - RTCIceServer( - urls="turn:openrelay.metered.ca:443?transport=tcp", - username="openrelayproject", - credential="openrelayproject", - ), - ] + if self.config.skip_openrelay: + ice_servers = [ + RTCIceServer(urls="stun:stun.l.google.com:19302"), + ] + else: + ice_servers = [ + RTCIceServer(urls="stun:stun.l.google.com:19302"), + RTCIceServer(urls="stun:openrelay.metered.ca:80"), + RTCIceServer( + urls="turn:openrelay.metered.ca:80", username="openrelayproject", credential="openrelayproject" + ), + RTCIceServer( + urls="turn:openrelay.metered.ca:443", username="openrelayproject", credential="openrelayproject" + ), + RTCIceServer( + urls="turn:openrelay.metered.ca:443?transport=tcp", + username="openrelayproject", + credential="openrelayproject", + ), + ] - pc = RTCPeerConnection(configuration=RTCConfiguration(iceServers=ice_servers)) + pc = RTCPeerConnection(configuration=RTCConfiguration(iceServers=ice_servers)) self.pcs.add(pc) video = SingleFrameStreamTrack() From 59f71eda0e0d1d29243d95b6f071ea8262ff4a66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=C3=A1=C5=A1=20Kulh=C3=A1nek?= Date: Mon, 6 Feb 2023 22:25:33 +0100 Subject: [PATCH 24/30] Allow methods configs to be discovered from external packages. (#1357) * Allow method_config to be registered from external packages * Add license * Add MethodSpecification class + fix linter errors --------- --- docs/developer_guides/config.md | 30 ++++++++++++++++ nerfstudio/configs/method_configs.py | 5 +++ nerfstudio/plugins/__init__.py | 13 +++++++ nerfstudio/plugins/registry.py | 50 +++++++++++++++++++++++++++ nerfstudio/plugins/types.py | 33 ++++++++++++++++++ pyproject.toml | 1 + tests/plugins/test_registry.py | 51 ++++++++++++++++++++++++++++ 7 files changed, 183 insertions(+) create mode 100644 nerfstudio/plugins/__init__.py create mode 100644 nerfstudio/plugins/registry.py create mode 100644 nerfstudio/plugins/types.py create mode 100644 tests/plugins/test_registry.py diff --git a/docs/developer_guides/config.md b/docs/developer_guides/config.md index bf216278f..29d093f9d 100644 --- a/docs/developer_guides/config.md +++ b/docs/developer_guides/config.md @@ -118,3 +118,33 @@ Often times, you just want to play with the parameters of an existing model with # NOTE: the dataparser and associated configurations go at the end of the command ns-train {METHOD_NAME} --vis viewer {DATA_PARSER} --scale-factor 0.5 ``` + +### Extending Nerfstudio with custom methods +In order to extend the Nerfstudio and register your own methods, you can package your code as a python package +and register it with Nerfstudio as a `nerfstudio.method_configs` entrypoint in the `pyproject.toml` file. +The Nerfstudio will automatically look for all registered methods and will register them to be used +by method such as `ns-train`. + +Here is an example: +```python +"""my_package/my_config.py""" + +from nerfstudio.engine.trainer import TrainerConfig +from nerfstudio.plugins.types import MethodSpecification + +MyMethod = MethodSpecification( + config=TrainerConfig( + method_name="my-method", + ... + ), + description="Custom description" +) + +"""pyproject.toml""" +[project] +name = "my_package" +... + +[project.entry-points.'nerfstudio.method_configs'] +my-method = 'my_package.my_config:MyMethod' +``` \ No newline at end of file diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index 947dad880..4f41ba4af 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -57,6 +57,7 @@ from nerfstudio.models.tensorf import TensoRFModelConfig from nerfstudio.models.vanilla_nerf import NeRFModel, VanillaModelConfig from nerfstudio.pipelines.base_pipeline import VanillaPipelineConfig from nerfstudio.pipelines.dynamic_batch import DynamicBatchPipelineConfig +from nerfstudio.plugins.registry import discover_methods method_configs: Dict[str, TrainerConfig] = {} descriptions = { @@ -387,6 +388,10 @@ method_configs["nerfplayer-ngp"] = TrainerConfig( vis="viewer", ) +external_methods, external_descriptions = discover_methods() +method_configs.update(external_methods) +descriptions.update(external_descriptions) + AnnotatedBaseConfigUnion = tyro.conf.SuppressFixed[ # Don't show unparseable (fixed) arguments in helptext. tyro.conf.FlagConversionOff[ tyro.extras.subcommand_type_from_defaults(defaults=method_configs, descriptions=descriptions) diff --git a/nerfstudio/plugins/__init__.py b/nerfstudio/plugins/__init__.py new file mode 100644 index 000000000..d926cf95c --- /dev/null +++ b/nerfstudio/plugins/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nerfstudio/plugins/registry.py b/nerfstudio/plugins/registry.py new file mode 100644 index 000000000..365f05a12 --- /dev/null +++ b/nerfstudio/plugins/registry.py @@ -0,0 +1,50 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Module that keeps all registered plugins and allows for plugin discovery. +""" + +import sys +import typing as t + +from rich.progress import Console + +from nerfstudio.plugins.types import MethodSpecification + +if sys.version_info < (3, 10): + from importlib_metadata import entry_points +else: + from importlib.metadata import entry_points +CONSOLE = Console(width=120) + + +def discover_methods(): + """ + Discovers all methods registered using the `nerfstudio.method_configs` entrypoint. + """ + methods = {} + descriptions = {} + discovered_entry_points = entry_points(group="nerfstudio.method_configs") + for name in discovered_entry_points.names: + specification = discovered_entry_points[name].load() + if not isinstance(specification, MethodSpecification): + CONSOLE.print( + "[bold yellow]Warning: Could not entry point {n} as it is not an instance of MethodSpecification" + ) + continue + specification = t.cast(MethodSpecification, specification) + methods[specification.config.method_name] = specification.config + descriptions[specification.config.method_name] = specification.description + return methods, descriptions diff --git a/nerfstudio/plugins/types.py b/nerfstudio/plugins/types.py new file mode 100644 index 000000000..ee7e8e470 --- /dev/null +++ b/nerfstudio/plugins/types.py @@ -0,0 +1,33 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This package contains specifications used to register plugins. +""" +from dataclasses import dataclass + +from nerfstudio.engine.trainer import TrainerConfig + + +@dataclass +class MethodSpecification: + """ + Method specification class used to register custom methods with Nerfstudio. + The registered methods will be available in commands such as `ns-train` + """ + + config: TrainerConfig + """Trainer configuration""" + description: str + """Method description shown in `ns-train` help""" diff --git a/pyproject.toml b/pyproject.toml index 21bc69456..ba57c01d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "functorch==0.2.1", "h5py>=2.9.0", "imageio>=2.21.1", + 'importlib-metadata>=6.0.0; python_version < "3.10"', "ipywidgets>=7.6", "jupyterlab>=3.3.4", "matplotlib>=3.5.3", diff --git a/tests/plugins/test_registry.py b/tests/plugins/test_registry.py new file mode 100644 index 000000000..fe8361a18 --- /dev/null +++ b/tests/plugins/test_registry.py @@ -0,0 +1,51 @@ +""" +Tests for the nerfstudio.plugins.registry module. +""" +import sys + +from nerfstudio.engine.trainer import TrainerConfig +from nerfstudio.pipelines.base_pipeline import VanillaPipelineConfig +from nerfstudio.plugins import registry +from nerfstudio.plugins.types import MethodSpecification + +if sys.version_info < (3, 10): + import importlib_metadata +else: + from importlib import metadata as importlib_metadata + + +TestConfig = MethodSpecification( + config=TrainerConfig( + method_name="test-method", + pipeline=VanillaPipelineConfig(), + optimizers={}, + ), + description="Test description", +) + + +def test_discover_methods(): + """Tests if a custom method gets properly registered using the discover_methods method""" + entry_points_backup = registry.entry_points + + def entry_points(group=None): + return importlib_metadata.EntryPoints( + [ + importlib_metadata.EntryPoint( + name="test", value="test_registry:TestConfig", group="nerfstudio.method_configs" + ) + ] + ).select(group=group) + + try: + # Mock importlib entry_points + registry.entry_points = entry_points + + # Discover plugins + methods, _ = registry.discover_methods() + assert "test-method" in methods + config = methods["test-method"] + assert isinstance(config, TrainerConfig) + finally: + # Revert mock + registry.entry_points = entry_points_backup From e749fbcf78e8fc9a1989b1a0d8dd966fe37fcc12 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Mon, 6 Feb 2023 18:33:59 -0800 Subject: [PATCH 25/30] Trace Stable Diffusion (#1362) * Trace Stable Diffusion * pylint Co-authored by Terrance Wang --- nerfstudio/generative/stable_diffusion.py | 60 ++++++++++------ scripts/generative/__init__.py | 0 scripts/generative/trace_stable_diffusion.py | 74 ++++++++++++++++++++ 3 files changed, 113 insertions(+), 21 deletions(-) create mode 100644 scripts/generative/__init__.py create mode 100644 scripts/generative/trace_stable_diffusion.py diff --git a/nerfstudio/generative/stable_diffusion.py b/nerfstudio/generative/stable_diffusion.py index af3aa2196..a91c3b49f 100644 --- a/nerfstudio/generative/stable_diffusion.py +++ b/nerfstudio/generative/stable_diffusion.py @@ -17,9 +17,11 @@ # Modified from https://github.com/ashawkey/stable-dreamfusion/blob/main/nerf/sd.py import sys +from dataclasses import dataclass from pathlib import Path from typing import List, Optional, Tuple, Union +import appdirs import mediapy import numpy as np import torch @@ -49,13 +51,18 @@ SD_SOURCE = "runwayml/stable-diffusion-v1-5" CLIP_SOURCE = "openai/clip-vit-large-patch14" +@dataclass +class UNet2DConditionOutput: + """Class to hold traced model""" + + sample: torch.FloatTensor + + class StableDiffusion(nn.Module): """Stable Diffusion implementation - Args: device: device to use num_train_timesteps: number of training timesteps - """ def __init__(self, device: Union[torch.device, str], num_train_timesteps: int = 1000) -> None: @@ -81,24 +88,46 @@ class StableDiffusion(nn.Module): pipe.enable_attention_slicing() + # use jitted unet + unet_traced_filename = Path(appdirs.user_data_dir("nerfstudio")) / "sd_unet_traced.pt" + if unet_traced_filename.exists(): + CONSOLE.print("Loading traced UNet.") + unet_traced = torch.jit.load(unet_traced_filename) + + class TracedUNet(torch.nn.Module): + """Jitted version of UNet""" + + def __init__(self): + super().__init__() + self.in_channels = pipe.unet.in_channels + self.device = pipe.unet.device + + def forward(self, latent_model_input, t, encoder_hidden_states): # pylint: disable=no-self-use + """Forward pass""" + sample = unet_traced(latent_model_input, t, encoder_hidden_states)[0] + return UNet2DConditionOutput(sample=sample) + + self.unet = TracedUNet() + del pipe.unet + else: + CONSOLE.print("[bold yellow] Warning: Loading UNet without JIT acceleration.") + CONSOLE.print(r"Run [yellow]python scripts/generative/trace_stable_diffusion.py[/yellow] for a speedup!") + self.unet = pipe.unet + self.unet.to(memory_format=torch.channels_last) + self.tokenizer = pipe.tokenizer - self.unet = pipe.unet self.text_encoder = pipe.text_encoder self.auto_encoder = pipe.vae - self.unet.to(memory_format=torch.channels_last) - CONSOLE.print("Stable Diffusion loaded!") def get_text_embeds( self, prompt: Union[str, List[str]], negative_prompt: Union[str, List[str]] ) -> TensorType[2, "max_length", "embed_dim"]: """Get text embeddings for prompt and negative prompt - Args: prompt: Prompt text negative_prompt: Negative prompt text - Returns: Text embeddings """ @@ -135,12 +164,10 @@ class StableDiffusion(nn.Module): guidance_scale: float = 100.0, ) -> Tuple[torch.Tensor, TensorType["BS", 4, "H", "W"], TensorType["BS", 4, "H", "W"]]: """Score Distilation Sampling loss proposed in DreamFusion paper (https://dreamfusion3d.github.io/) - Args: text_embeddings: Text embeddings image: Rendered image guidance_scale: How much to weigh the guidance - Returns: A tuple of Loss, latent, and gradient """ @@ -181,7 +208,6 @@ class StableDiffusion(nn.Module): latents: Optional[TensorType["BS", 4, "H", "W"]] = None, ) -> TensorType["BS", 4, "H", "W"]: """Produce latents for a given text embedding - Args: text_embeddings: Text embeddings height: Height of the image @@ -189,7 +215,6 @@ class StableDiffusion(nn.Module): num_inference_steps: Number of inference steps guidance_scale: How much to weigh the guidance latents: Latents to start with - Returns: Latents """ @@ -209,7 +234,9 @@ class StableDiffusion(nn.Module): # predict the noise residual with torch.no_grad(): - noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings)["sample"] + noise_pred = self.unet( + latent_model_input, t.to(self.device), encoder_hidden_states=text_embeddings + ).sample # perform guidance noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) @@ -221,10 +248,8 @@ class StableDiffusion(nn.Module): def latents_to_img(self, latents: TensorType["BS", 4, "H", "W"]) -> TensorType["BS", 3, "H", "W"]: """Convert latents to images - Args: latents: Latents to convert - Returns: Images """ @@ -240,10 +265,8 @@ class StableDiffusion(nn.Module): def imgs_to_latent(self, imgs: TensorType["BS", 3, "H", "W"]) -> TensorType["BS", 4, "H", "W"]: """Convert images to latents - Args: imgs: Images to convert - Returns: Latents """ @@ -263,14 +286,12 @@ class StableDiffusion(nn.Module): latents=None, ) -> np.ndarray: """Generate an images from a prompts. - Args: prompts: The prompt to generate an image from. negative_prompts: The negative prompt to generate an image from. num_inference_steps: The number of inference steps to perform. guidance_scale: The scale of the guidance. latents: The latents to start from, defaults to random. - Returns: The generated image. """ @@ -297,14 +318,12 @@ class StableDiffusion(nn.Module): self, prompts, negative_prompts="", num_inference_steps=50, guidance_scale=7.5, latents=None ) -> np.ndarray: """Generate an image from a prompt. - Args: prompts: The prompt to generate an image from. negative_prompts: The negative prompt to generate an image from. num_inference_steps: The number of inference steps to perform. guidance_scale: The scale of the guidance. latents: The latents to start from, defaults to random. - Returns: The generated image. """ @@ -315,7 +334,6 @@ def generate_image( prompt: str, negative: str = "", seed: int = 0, steps: int = 50, save_path: Path = Path("test_sd.png") ): """Generate an image from a prompt using Stable Diffusion. - Args: prompt: The prompt to use. negative: The negative prompt to use. diff --git a/scripts/generative/__init__.py b/scripts/generative/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/generative/trace_stable_diffusion.py b/scripts/generative/trace_stable_diffusion.py new file mode 100644 index 000000000..a501bd424 --- /dev/null +++ b/scripts/generative/trace_stable_diffusion.py @@ -0,0 +1,74 @@ +"""Trace Stable Diffusion UNet for speed improvement""" + +import functools +import sys +from pathlib import Path + +import appdirs +import torch +import tyro +from rich.console import Console + +CONSOLE = Console(width=120) + +try: + from diffusers import StableDiffusionPipeline +except ImportError: + CONSOLE.print("[bold red]Missing Stable Diffusion packages.") + CONSOLE.print(r"Install using [yellow]pip install nerfstudio\[gen][/yellow]") + CONSOLE.print(r"or [yellow]pip install -e .\[gen][/yellow] if installing from source.") + sys.exit(1) + + +def jit_unet(save_dir: Path = Path(appdirs.user_data_dir("nerfstudio"))): + """Trace Stable Diffusion UNet for speed improvement + + Args: + save_dir: directory to save the traced model + """ + + # torch disable grad + torch.set_grad_enabled(False) + + # load inputs + def generate_inputs(): + sample = torch.randn(2, 4, 64, 64).half().cuda() + timestep = torch.rand(1).half().cuda() * 999 + encoder_hidden_states = torch.randn(2, 77, 768).half().cuda() + return sample, timestep, encoder_hidden_states + + pipe = StableDiffusionPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", + torch_dtype=torch.float16, + ).to("cuda") + + pipe.enable_attention_slicing() + + unet = pipe.unet + unet.eval() + unet.to(memory_format=torch.channels_last) # use channels_last memory format + unet.forward = functools.partial(unet.forward, return_dict=False) # set return_dict=False as default + + # warmup + inputs = None + for _ in range(3): + with torch.inference_mode(): + inputs = generate_inputs() + _ = unet(*inputs) + + # trace + print("tracing..") + assert inputs is not None + unet_traced = torch.jit.trace(unet, inputs) + unet_traced.eval() + print("done tracing") + + # save the traced model + unet_traced_filename = save_dir / "sd_unet_traced.pt" + if not unet_traced_filename.exists(): + unet_traced_filename.parent.mkdir(parents=True, exist_ok=True) + unet_traced.save(unet_traced_filename) + + +if __name__ == "__main__": + tyro.cli(jit_unet) From 6136745d75b484bb2306fb99ef39d966daaf6df6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=C3=A1=C5=A1=20Kulh=C3=A1nek?= Date: Tue, 7 Feb 2023 18:49:09 +0100 Subject: [PATCH 26/30] Fix ns-train help (#1370) --- nerfstudio/engine/trainer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index 2d4a45395..d0e750c64 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -77,8 +77,9 @@ class TrainerConfig(ExperimentConfig): load_step: Optional[int] = None """Optionally specify model step to load from; if none, will find most recent model in load_dir.""" load_config: Optional[Path] = None - """Optionally log gradients during training""" + """Path to config YAML file.""" log_gradients: bool = False + """Optionally log gradients during training""" class Trainer: From 2ff1d3b19ecf8d9c04c629daae17bfc246bd1b1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=C3=A1=C5=A1=20Kulh=C3=A1nek?= Date: Tue, 7 Feb 2023 22:08:14 +0100 Subject: [PATCH 27/30] Improve ns-process-data images when using existing COLMAP model (#1371) * Allow skipping copy+downscale of images in ns-process-data, fix bug when using --skip-colmap on images.bin with non-standard image names * Rename flags and improve error messages --------- Co-authored-by: Matthew Tancik --- nerfstudio/process_data/colmap_utils.py | 11 +- nerfstudio/process_data/process_data_utils.py | 19 ++- scripts/process_data.py | 108 ++++++++++++------ 3 files changed, 98 insertions(+), 40 deletions(-) diff --git a/nerfstudio/process_data/colmap_utils.py b/nerfstudio/process_data/colmap_utils.py index 9db20647f..93f47adfa 100644 --- a/nerfstudio/process_data/colmap_utils.py +++ b/nerfstudio/process_data/colmap_utils.py @@ -604,6 +604,14 @@ def colmap_to_json( cameras = read_cameras_binary(cameras_path) images = read_images_binary(images_path) + # Images were renamed to frame_{i:05d}.{ext} and + # the filenames needs to be replaced in the transforms.json as well + original_filenames = [x.name for x in images.values()] + # Sort was used in nerfstudio.process_data.process_data_utils:get_image_filenames + original_filenames.sort() + # Build the map to the new filenames + filename_map = {name: f"frame_{i+1:05d}{os.path.splitext(name)[-1]}" for i, name in enumerate(original_filenames)} + # Only supports one camera camera_params = cameras[1].params @@ -619,7 +627,8 @@ def colmap_to_json( c2w = c2w[np.array([1, 0, 2, 3]), :] c2w[2, :] *= -1 - name = Path(f"./images/{im_data.name}") + name = filename_map[im_data.name] + name = Path(f"./images/{name}") frame = { "file_path": name.as_posix(), diff --git a/nerfstudio/process_data/process_data_utils.py b/nerfstudio/process_data/process_data_utils.py index cedfc2b27..4919edd93 100644 --- a/nerfstudio/process_data/process_data_utils.py +++ b/nerfstudio/process_data/process_data_utils.py @@ -46,6 +46,19 @@ CAMERA_MODELS = { } +def list_images(data: Path) -> List[Path]: + """Lists all supported images in a directory + + Args: + data: Path to the directory of images. + Returns: + Paths to images contained in the directory + """ + allowed_exts = [".jpg", ".jpeg", ".png", ".tif", ".tiff"] + image_paths = sorted([p for p in data.glob("[!.]*") if p.suffix.lower() in allowed_exts]) + return image_paths + + def get_image_filenames(directory: Path, max_num_images: int = -1) -> Tuple[List[Path], int]: """Returns a list of image filenames in a directory. @@ -55,8 +68,7 @@ def get_image_filenames(directory: Path, max_num_images: int = -1) -> Tuple[List Returns: A tuple of A list of image filenames, number of original image paths. """ - allowed_exts = [".jpg", ".jpeg", ".png", ".tif", ".tiff"] - image_paths = sorted([p for p in directory.glob("[!.]*") if p.suffix.lower() in allowed_exts]) + image_paths = list_images(directory) num_orig_images = len(image_paths) if max_num_images != -1 and num_orig_images > max_num_images: @@ -240,8 +252,7 @@ def copy_images(data: Path, image_dir: Path, verbose) -> int: The number of images copied. """ with status(msg="[bold yellow]Copying images...", spinner="bouncingBall", verbose=verbose): - allowed_exts = [".jpg", ".jpeg", ".png", ".tif", ".tiff"] - image_paths = sorted([p for p in data.glob("[!.]*") if p.suffix.lower() in allowed_exts]) + image_paths = list_images(data) if len(image_paths) == 0: CONSOLE.log("[bold red]:skull: No usable images in the data folder.") diff --git a/scripts/process_data.py b/scripts/process_data.py index 4d8e9acf7..f6f9a9a7f 100755 --- a/scripts/process_data.py +++ b/scripts/process_data.py @@ -28,6 +28,7 @@ from nerfstudio.process_data.process_data_utils import CAMERA_MODELS from nerfstudio.utils import install_checks CONSOLE = Console(width=120) +DEFAULT_COLMAP_PATH = Path("colmap/sparse/0") @dataclass @@ -74,6 +75,12 @@ class ProcessImages: will downscale the images by 2x, 4x, and 8x.""" skip_colmap: bool = False """If True, skips COLMAP and generates transforms.json if possible.""" + skip_image_processing: bool = False + """If True, skips copying and downscaling of images and only runs COLMAP if possible and enabled""" + colmap_model_path: Path = DEFAULT_COLMAP_PATH + """Optionally sets the path of the colmap model. Used only when --skip-colmap is set to True. + The path is relative to the output directory. + """ colmap_cmd: str = "colmap" """How to call the COLMAP executable.""" gpu: bool = True @@ -83,6 +90,19 @@ class ProcessImages: def main(self) -> None: """Process images into a nerfstudio dataset.""" + require_cameras_exist = False + colmap_model_path = self.output_dir / Path(self.colmap_model_path) + if self.colmap_model_path != DEFAULT_COLMAP_PATH: + if not self.skip_colmap: + CONSOLE.log("[bold red]The --colmap-model-path can only be used when --skip-colmap is not set.") + sys.exit(1) + elif not (self.output_dir / self.colmap_model_path).exists(): + CONSOLE.log( + f"[bold red]The colmap-model-path {self.output_dir / self.colmap_model_path} does not exist." + ) + sys.exit(1) + require_cameras_exist = True + install_checks.check_ffmpeg_installed() install_checks.check_colmap_installed() @@ -92,57 +112,46 @@ class ProcessImages: summary_log = [] - # Copy images to output directory - num_frames = process_data_utils.copy_images(self.data, image_dir=image_dir, verbose=self.verbose) - summary_log.append(f"Starting with {num_frames} images") + # Copy and downscale images + if not self.skip_image_processing: + # Copy images to output directory + num_frames = process_data_utils.copy_images(self.data, image_dir=image_dir, verbose=self.verbose) + summary_log.append(f"Starting with {num_frames} images") - # Downscale images - summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose)) + # Downscale images + summary_log.append( + process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose) + ) + else: + num_frames = len(process_data_utils.list_images(self.data)) + if num_frames == 0: + CONSOLE.log("[bold red]:skull: No usable images in the data folder.") + sys.exit(1) + summary_log.append(f"Starting with {num_frames} images") # Run COLMAP colmap_dir = self.output_dir / "colmap" if not self.skip_colmap: colmap_dir.mkdir(parents=True, exist_ok=True) + colmap_model_path = colmap_dir / "sparse" / "0" + require_cameras_exist = True - (sfm_tool, feature_type, matcher_type) = process_data_utils.find_tool_feature_matcher_combination( - self.sfm_tool, self.feature_type, self.matcher_type - ) - - if sfm_tool == "colmap": - colmap_utils.run_colmap( - image_dir=image_dir, - colmap_dir=colmap_dir, - camera_model=CAMERA_MODELS[self.camera_type], - gpu=self.gpu, - verbose=self.verbose, - matching_method=self.matching_method, - colmap_cmd=self.colmap_cmd, - ) - elif sfm_tool == "hloc": - hloc_utils.run_hloc( - image_dir=image_dir, - colmap_dir=colmap_dir, - camera_model=CAMERA_MODELS[self.camera_type], - verbose=self.verbose, - matching_method=self.matching_method, - feature_type=feature_type, - matcher_type=matcher_type, - ) - else: - CONSOLE.log("[bold red]Invalid combination of sfm_tool, feature_type, and matcher_type, exiting") - sys.exit(1) + self._run_colmap(image_dir, colmap_dir) # Save transforms.json - if (colmap_dir / "sparse" / "0" / "cameras.bin").exists(): + if (colmap_model_path / "cameras.bin").exists(): with CONSOLE.status("[bold yellow]Saving results to transforms.json", spinner="balloon"): num_matched_frames = colmap_utils.colmap_to_json( - cameras_path=colmap_dir / "sparse" / "0" / "cameras.bin", - images_path=colmap_dir / "sparse" / "0" / "images.bin", + cameras_path=colmap_model_path / "cameras.bin", + images_path=colmap_model_path / "images.bin", output_dir=self.output_dir, camera_model=CAMERA_MODELS[self.camera_type], ) summary_log.append(f"Colmap matched {num_matched_frames} images") summary_log.append(colmap_utils.get_matching_summary(num_frames, num_matched_frames)) + elif require_cameras_exist: + CONSOLE.log(f"[bold red]Could not find existing COLMAP results ({colmap_model_path / 'cameras.bin'}).") + sys.exit(1) else: CONSOLE.log("[bold yellow]Warning: could not find existing COLMAP results. Not generating transforms.json") @@ -152,6 +161,35 @@ class ProcessImages: CONSOLE.print(summary, justify="center") CONSOLE.rule() + def _run_colmap(self, image_dir, colmap_dir): + (sfm_tool, feature_type, matcher_type) = process_data_utils.find_tool_feature_matcher_combination( + self.sfm_tool, self.feature_type, self.matcher_type + ) + + if sfm_tool == "colmap": + colmap_utils.run_colmap( + image_dir=image_dir, + colmap_dir=colmap_dir, + camera_model=CAMERA_MODELS[self.camera_type], + gpu=self.gpu, + verbose=self.verbose, + matching_method=self.matching_method, + colmap_cmd=self.colmap_cmd, + ) + elif sfm_tool == "hloc": + hloc_utils.run_hloc( + image_dir=image_dir, + colmap_dir=colmap_dir, + camera_model=CAMERA_MODELS[self.camera_type], + verbose=self.verbose, + matching_method=self.matching_method, + feature_type=feature_type, + matcher_type=matcher_type, + ) + else: + CONSOLE.log("[bold red]Invalid combination of sfm_tool, feature_type, and matcher_type, exiting") + sys.exit(1) + @dataclass class ProcessVideo: From 8c9c2ba2ab052317ff47f2f93d0ff5e0c7655471 Mon Sep 17 00:00:00 2001 From: Yiming Zhou <107962411+yimingzhou1@users.noreply.github.com> Date: Tue, 7 Feb 2023 16:26:05 -0800 Subject: [PATCH 28/30] Add support for sparse depth maps (#1373) Add support for sparse depth maps (#2) * sparse depth maps * pr comments * unified loss * minor * lint --- nerfstudio/model_components/losses.py | 18 ++++++++++++------ nerfstudio/models/depth_nerfacto.py | 9 ++++++--- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/nerfstudio/model_components/losses.py b/nerfstudio/model_components/losses.py index 6a981d664..353dd7db9 100644 --- a/nerfstudio/model_components/losses.py +++ b/nerfstudio/model_components/losses.py @@ -34,7 +34,7 @@ EPS = 1.0e-7 URF_SIGMA_SCALE_FACTOR = 3.0 -class DephtLossType(Enum): +class DepthLossType(Enum): """Types of depth losses for depth supervision.""" DS_NERF = 1 @@ -225,8 +225,11 @@ def ds_nerf_depth_loss( Returns: Depth loss scalar. """ + depth_mask = termination_depth > 0 + loss = -torch.log(weights + EPS) * torch.exp(-((steps - termination_depth[:, None]) ** 2) / (2 * sigma)) * lengths - return torch.mean(loss.sum(-2)) + loss = loss.sum(-2) * depth_mask + return torch.mean(loss) def urban_radiance_field_depth_loss( @@ -247,6 +250,8 @@ def urban_radiance_field_depth_loss( Returns: Depth loss scalar. """ + depth_mask = termination_depth > 0 + # Expected depth loss expected_depth_loss = (termination_depth - predicted_depth) ** 2 @@ -262,7 +267,8 @@ def urban_radiance_field_depth_loss( line_of_sight_loss_empty = (line_of_sight_loss_empty_mask * weights**2).sum(-2) line_of_sight_loss = line_of_sight_loss_near + line_of_sight_loss_empty - return torch.mean(expected_depth_loss + line_of_sight_loss) + loss = (expected_depth_loss + line_of_sight_loss) * depth_mask + return torch.mean(loss) def depth_loss( @@ -273,7 +279,7 @@ def depth_loss( sigma: TensorType[0], directions_norm: TensorType[..., 1], is_euclidean: bool, - depth_loss_type: DephtLossType, + depth_loss_type: DepthLossType, ) -> TensorType[0]: """Implementation of depth losses. @@ -294,11 +300,11 @@ def depth_loss( termination_depth = termination_depth * directions_norm steps = (ray_samples.frustums.starts + ray_samples.frustums.ends) / 2 - if depth_loss_type == DephtLossType.DS_NERF: + if depth_loss_type == DepthLossType.DS_NERF: lengths = ray_samples.frustums.ends - ray_samples.frustums.starts return ds_nerf_depth_loss(weights, termination_depth, steps, lengths, sigma) - if depth_loss_type == DephtLossType.URF: + if depth_loss_type == DepthLossType.URF: return urban_radiance_field_depth_loss(weights, termination_depth, predicted_depth, steps, sigma) raise NotImplementedError("Provided depth loss type not implemented.") diff --git a/nerfstudio/models/depth_nerfacto.py b/nerfstudio/models/depth_nerfacto.py index ac0768fdf..2bea84c7b 100644 --- a/nerfstudio/models/depth_nerfacto.py +++ b/nerfstudio/models/depth_nerfacto.py @@ -24,7 +24,7 @@ from typing import Dict, Tuple, Type import torch from nerfstudio.cameras.rays import RayBundle -from nerfstudio.model_components.losses import DephtLossType, depth_loss +from nerfstudio.model_components.losses import DepthLossType, depth_loss from nerfstudio.models.nerfacto import NerfactoModel, NerfactoModelConfig from nerfstudio.utils import colormaps @@ -46,7 +46,7 @@ class DepthNerfactoModelConfig(NerfactoModelConfig): """Starting uncertainty around depth values in meters (defaults to 0.2m).""" sigma_decay_rate: float = 0.99985 """Rate of exponential decay.""" - depth_loss_type: DephtLossType = DephtLossType.DS_NERF + depth_loss_type: DepthLossType = DepthLossType.DS_NERF """Depth loss type.""" @@ -118,7 +118,10 @@ class DepthNerfactoModel(NerfactoModel): far_plane=torch.max(ground_truth_depth), ) images["depth"] = torch.cat([ground_truth_depth_colormap, predicted_depth_colormap], dim=1) - metrics["depth_mse"] = torch.nn.functional.mse_loss(outputs["depth"], ground_truth_depth) + depth_mask = ground_truth_depth > 0 + metrics["depth_mse"] = torch.nn.functional.mse_loss( + outputs["depth"][depth_mask], ground_truth_depth[depth_mask] + ) return metrics, images def _get_sigma(self): From 92cadf5ce42a927dc06b485a9f4dce6910fc6a8d Mon Sep 17 00:00:00 2001 From: NCJ Date: Thu, 9 Feb 2023 02:33:23 +0800 Subject: [PATCH 29/30] Fix Multiple Typos (#1376) fix multiple typos --- nerfstudio/cameras/rays.py | 6 +++--- nerfstudio/engine/trainer.py | 2 +- nerfstudio/fields/instant_ngp_field.py | 2 +- nerfstudio/model_components/renderers.py | 4 ++-- nerfstudio/utils/math.py | 2 +- scripts/train.py | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/nerfstudio/cameras/rays.py b/nerfstudio/cameras/rays.py index 8718712fb..c65315c26 100644 --- a/nerfstudio/cameras/rays.py +++ b/nerfstudio/cameras/rays.py @@ -44,7 +44,7 @@ class Frustums(TensorDataclass): """Offsets for each sample position""" def get_positions(self) -> TensorType[..., 3]: - """Calulates "center" position of frustum. Not weighted by mass. + """Calculates "center" position of frustum. Not weighted by mass. Returns: xyz positions. @@ -61,7 +61,7 @@ class Frustums(TensorDataclass): def get_gaussian_blob(self) -> Gaussians: """Calculates guassian approximation of conical frustum. - Resturns: + Returns: Conical frustums approximated by gaussian distribution. """ # Cone radius is set such that the square pixel_area matches the cone area. @@ -109,7 +109,7 @@ class RaySamples(TensorDataclass): spacing_to_euclidean_fn: Optional[Callable] = None """Function to convert bins to euclidean distance.""" metadata: Optional[Dict[str, TensorType["bs":..., "latent_dims"]]] = None - """addtional information relevant to generating ray samples""" + """additional information relevant to generating ray samples""" times: Optional[TensorType[..., 1]] = None """Times at which rays are sampled""" diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index d0e750c64..ac524e7b3 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -292,7 +292,7 @@ class Trainer: @check_viewer_enabled def _update_viewer_rays_per_sec(self, train_t: TimeWriter, vis_t: TimeWriter, step: int): - """Performs update on rays/sec calclation for training + """Performs update on rays/sec calculation for training Args: train_t: timer object carrying time to execute total training iteration diff --git a/nerfstudio/fields/instant_ngp_field.py b/nerfstudio/fields/instant_ngp_field.py index 18a806b68..846ea121e 100644 --- a/nerfstudio/fields/instant_ngp_field.py +++ b/nerfstudio/fields/instant_ngp_field.py @@ -58,7 +58,7 @@ class TCNNInstantNGPField(Field): num_layers_color: number of hidden layers for color network hidden_dim_color: dimension of hidden layers for color network use_appearance_embedding: whether to use appearance embedding - num_images: number of images, requried if use_appearance_embedding is True + num_images: number of images, required if use_appearance_embedding is True appearance_embedding_dim: dimension of appearance embedding contraction_type: type of contraction num_levels: number of levels of the hashmap for the base mlp diff --git a/nerfstudio/model_components/renderers.py b/nerfstudio/model_components/renderers.py index d13c29233..cf5af2a7d 100644 --- a/nerfstudio/model_components/renderers.py +++ b/nerfstudio/model_components/renderers.py @@ -55,7 +55,7 @@ def background_color_override_context(mode: TensorType[3]) -> Generator[None, No class RGBRenderer(nn.Module): - """Standard volumetic rendering. + """Standard volumetric rendering. Args: background_color: Background color as RGB. Uses random colors if None. @@ -165,7 +165,7 @@ class SHRenderer(nn.Module): """Composite samples along ray and render color image Args: - sh: Spherical hamonics coefficients for each sample + sh: Spherical harmonics coefficients for each sample directions: Sample direction weights: Weights for each sample diff --git a/nerfstudio/utils/math.py b/nerfstudio/utils/math.py index 937866dcc..cee6108a9 100644 --- a/nerfstudio/utils/math.py +++ b/nerfstudio/utils/math.py @@ -106,7 +106,7 @@ def compute_3d_gaussian( dir_variance: TensorType[..., 1], radius_variance: TensorType[..., 1], ) -> Gaussians: - """Compute guassian along ray. + """Compute gaussian along ray. Args: directions: Axis of Gaussian. diff --git a/scripts/train.py b/scripts/train.py index 240fdeca0..f19b0ccc5 100755 --- a/scripts/train.py +++ b/scripts/train.py @@ -152,7 +152,7 @@ def launch( config: Optional[TrainerConfig] = None, timeout: timedelta = DEFAULT_TIMEOUT, ) -> None: - """Function that spawns muliple processes to call on main_func + """Function that spawns multiple processes to call on main_func Args: main_func (Callable): function that will be called by the distributed workers From df784e96e7979aaa4320284c087d7036dce67c28 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Wed, 8 Feb 2023 11:23:34 -0800 Subject: [PATCH 30/30] Colmap docs note (#1380) --- docs/quickstart/custom_dataset.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/quickstart/custom_dataset.md b/docs/quickstart/custom_dataset.md index a7fe05a05..ebc6ce313 100644 --- a/docs/quickstart/custom_dataset.md +++ b/docs/quickstart/custom_dataset.md @@ -32,6 +32,12 @@ To assist running on custom data we have a script that will process a video or f - COLMAP can be finicky. Try your best to capture overlapping, non-blurry images. ::: +:::{admonition} Note +:class: warning + +- If COLMAP opens in a new window, simply close it to resume processing. + ::: + ### Processing Data ```bash