diff --git a/Dockerfile b/Dockerfile index c2e3ab82b..94587f550 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,8 @@ ENV DEBIAN_FRONTEND=noninteractive ## Set timezone as it is required by some packages. ENV TZ=Europe/Berlin ## CUDA architectures, required by tiny-cuda-nn. -ENV TCNN_CUDA_ARCHITECTURES=86 +## NOTE: All commonly used GPU architectures are included and supported here. To speedup the image build process remove all architectures but the one of your explicit GPU. Find details here: https://developer.nvidia.com/cuda-gpus (8.6 translates to 86 in the line below) or in the docs. +ENV TCNN_CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37 ## CUDA Home, required to find CUDA in some packages. ENV CUDA_HOME="/usr/local/cuda" diff --git a/docs/developer_guides/config.md b/docs/developer_guides/config.md index bf216278f..29d093f9d 100644 --- a/docs/developer_guides/config.md +++ b/docs/developer_guides/config.md @@ -118,3 +118,33 @@ Often times, you just want to play with the parameters of an existing model with # NOTE: the dataparser and associated configurations go at the end of the command ns-train {METHOD_NAME} --vis viewer {DATA_PARSER} --scale-factor 0.5 ``` + +### Extending Nerfstudio with custom methods +In order to extend the Nerfstudio and register your own methods, you can package your code as a python package +and register it with Nerfstudio as a `nerfstudio.method_configs` entrypoint in the `pyproject.toml` file. +The Nerfstudio will automatically look for all registered methods and will register them to be used +by method such as `ns-train`. + +Here is an example: +```python +"""my_package/my_config.py""" + +from nerfstudio.engine.trainer import TrainerConfig +from nerfstudio.plugins.types import MethodSpecification + +MyMethod = MethodSpecification( + config=TrainerConfig( + method_name="my-method", + ... + ), + description="Custom description" +) + +"""pyproject.toml""" +[project] +name = "my_package" +... + +[project.entry-points.'nerfstudio.method_configs'] +my-method = 'my_package.my_config:MyMethod' +``` \ No newline at end of file diff --git a/docs/developer_guides/viewer/viewer_overview.md b/docs/developer_guides/viewer/viewer_overview.md index b80463e75..4e10319b6 100644 --- a/docs/developer_guides/viewer/viewer_overview.md +++ b/docs/developer_guides/viewer/viewer_overview.md @@ -50,6 +50,10 @@ yarn install yarn start ``` +The local webserver runs on port 4000 by default, +so when `ns-train` is running, you can connect to the viewer locally at +[http://localhost:4000/?websocket_url=ws://localhost:7007](http://localhost:4000/?websocket_url=ws://localhost:7007) + #### Running the Bridge Server Manually The viewer bridge server runs on the same machine that you use for training. The training code will connect to this server with a lightweight TCP connection using ZMQ. The training job will launch the viewer bridge server is you specify `--viewer.launch-bridge-server` in the terminal. Otherwise, you can launch the bridge server manually with the following script. diff --git a/docs/quickstart/custom_dataset.md b/docs/quickstart/custom_dataset.md index a7fe05a05..ebc6ce313 100644 --- a/docs/quickstart/custom_dataset.md +++ b/docs/quickstart/custom_dataset.md @@ -32,6 +32,12 @@ To assist running on custom data we have a script that will process a video or f - COLMAP can be finicky. Try your best to capture overlapping, non-blurry images. ::: +:::{admonition} Note +:class: warning + +- If COLMAP opens in a new window, simply close it to resume processing. + ::: + ### Processing Data ```bash diff --git a/docs/quickstart/installation.md b/docs/quickstart/installation.md index ae6a17cc9..97c63aac8 100644 --- a/docs/quickstart/installation.md +++ b/docs/quickstart/installation.md @@ -95,13 +95,28 @@ docker run --gpus all -v /folder/of/your/data:/workspace/ -v /home//. ``` ### Note - The container works on Linux and Windows, depending on your OS some additional setup steps might be required to provide access to your GPU inside containers. -- Paths on Windows use backslash '\\\\' while unix based systems use a frontslash '/' for paths, where backslashes might require an escape character depending on where they are used (e.g. C:\\\\folder1\\\\folder2...). Ensure to use the correct paths when mounting folders or providing paths as parameters. +- Paths on Windows use backslash '\\' while unix based systems use a frontslash '/' for paths, where backslashes might require an escape character depending on where they are used (e.g. C:\\\\folder1\\\\folder2...). Alternatively, mounts can be quoted (e.g. ```-v 'C:\local_folder:/docker_folder'```). Ensure to use the correct paths when mounting folders or providing paths as parameters. +- Always use full paths, relative paths are known to create issues when being used in mounts into docker. - Everything inside the container, what is not in a mounted folder (workspace in the above example), will be permanently removed after destroying the container. Always do all your tasks and output folder in workdir! - The user inside the container is called 'user' and is mapped to the local user with ID 1000 (usually the first non-root user on Linux systems). - The container currently is based on nvidia/cuda:11.7.1-devel-ubuntu22.04, consequently it comes with CUDA 11.7 which must be supported by the nvidia driver. No local CUDA installation is required or will be affected by using the docker image. - The docker image (respectively Ubuntu 22.04) comes with Python3.10, no older version of Python is installed. - If you call the container with commands directly, you still might want to add the interactive terminal ('-it') flag to get live log outputs of the nerfstudio scripts. In case the container is used in an automated environment the flag should be discarded. +- The current version of docker is built for multi-architecture (CUDA architectures) use. The target architecture must be defined at build time for tinyCUDNN to be able to compile properly. If your GPU architecture is not covered by the following table you need to replace the number in the line ```ENV TCNN_CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37``` to you specific architecture. It also is a good idea to remove all architectures but yours (e.g. ```ENV TCNN_CUDA_ARCHITECTURES=86```) to speedup the docker build a lot. +**Currently supported CUDA architectures in the docker image** + +GPU | CUDA arch +-- | -- +H100 | 90 +40X0 | 89 +30X0 | 86 +A100 | 80 +20X0 | 75 +TITAN V / V100 | 70 +10X0 / TITAN Xp | 61 +9X0 | 52 +K80 | 37 ## Installation FAQ diff --git a/nerfstudio/cameras/cameras.py b/nerfstudio/cameras/cameras.py index 240c771b6..d7d228bae 100644 --- a/nerfstudio/cameras/cameras.py +++ b/nerfstudio/cameras/cameras.py @@ -16,7 +16,6 @@ Camera Models """ import base64 -import importlib import math import os from dataclasses import dataclass @@ -472,15 +471,7 @@ class Cameras(TensorDataclass): rays_o = rays_o.reshape((-1, 3)) rays_d = rays_d.reshape((-1, 3)) - if self._use_nerfacc: - try: - nerfacc = importlib.import_module("nerfacc") - t_min, t_max = nerfacc.ray_aabb_intersect(rays_o, rays_d, tensor_aabb) - except: # pylint: disable=bare-except - t_min, t_max = nerfstudio.utils.math.intersect_aabb(rays_o, rays_d, tensor_aabb) - self._use_nerfacc = False - else: - t_min, t_max = nerfstudio.utils.math.intersect_aabb(rays_o, rays_d, tensor_aabb) + t_min, t_max = nerfstudio.utils.math.intersect_aabb(rays_o, rays_d, tensor_aabb) t_min = t_min.reshape([shape[0], shape[1], 1]) t_max = t_max.reshape([shape[0], shape[1], 1]) diff --git a/nerfstudio/cameras/rays.py b/nerfstudio/cameras/rays.py index 8718712fb..c65315c26 100644 --- a/nerfstudio/cameras/rays.py +++ b/nerfstudio/cameras/rays.py @@ -44,7 +44,7 @@ class Frustums(TensorDataclass): """Offsets for each sample position""" def get_positions(self) -> TensorType[..., 3]: - """Calulates "center" position of frustum. Not weighted by mass. + """Calculates "center" position of frustum. Not weighted by mass. Returns: xyz positions. @@ -61,7 +61,7 @@ class Frustums(TensorDataclass): def get_gaussian_blob(self) -> Gaussians: """Calculates guassian approximation of conical frustum. - Resturns: + Returns: Conical frustums approximated by gaussian distribution. """ # Cone radius is set such that the square pixel_area matches the cone area. @@ -109,7 +109,7 @@ class RaySamples(TensorDataclass): spacing_to_euclidean_fn: Optional[Callable] = None """Function to convert bins to euclidean distance.""" metadata: Optional[Dict[str, TensorType["bs":..., "latent_dims"]]] = None - """addtional information relevant to generating ray samples""" + """additional information relevant to generating ray samples""" times: Optional[TensorType[..., 1]] = None """Times at which rays are sampled""" diff --git a/nerfstudio/configs/base_config.py b/nerfstudio/configs/base_config.py index 548324069..a2e695c14 100644 --- a/nerfstudio/configs/base_config.py +++ b/nerfstudio/configs/base_config.py @@ -23,6 +23,8 @@ from dataclasses import dataclass from pathlib import Path from typing import Any, List, Optional, Tuple, Type +from typing_extensions import Literal + # model instances from nerfstudio.utils import writer @@ -149,3 +151,7 @@ class ViewerConfig(PrintableConfig): skip_openrelay: bool = False """Avoid using openrelay to communicate with the viewer. Try disabling if you have trouble connecting to the viewer""" + codec: Literal["H264", "VP8"] = "VP8" + """Video codec that viewer will use.""" + local: bool = False + """If running local server instance, avoid using relays to communicate with the viewer.""" diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index 947dad880..4f41ba4af 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -57,6 +57,7 @@ from nerfstudio.models.tensorf import TensoRFModelConfig from nerfstudio.models.vanilla_nerf import NeRFModel, VanillaModelConfig from nerfstudio.pipelines.base_pipeline import VanillaPipelineConfig from nerfstudio.pipelines.dynamic_batch import DynamicBatchPipelineConfig +from nerfstudio.plugins.registry import discover_methods method_configs: Dict[str, TrainerConfig] = {} descriptions = { @@ -387,6 +388,10 @@ method_configs["nerfplayer-ngp"] = TrainerConfig( vis="viewer", ) +external_methods, external_descriptions = discover_methods() +method_configs.update(external_methods) +descriptions.update(external_descriptions) + AnnotatedBaseConfigUnion = tyro.conf.SuppressFixed[ # Don't show unparseable (fixed) arguments in helptext. tyro.conf.FlagConversionOff[ tyro.extras.subcommand_type_from_defaults(defaults=method_configs, descriptions=descriptions) diff --git a/nerfstudio/engine/optimizers.py b/nerfstudio/engine/optimizers.py index c97b10580..6878eccc2 100644 --- a/nerfstudio/engine/optimizers.py +++ b/nerfstudio/engine/optimizers.py @@ -18,7 +18,7 @@ Optimizers class. from __future__ import annotations from dataclasses import dataclass -from typing import Any, Dict, List, Type +from typing import Any, Dict, List, Optional, Type import torch from torch.cuda.amp.grad_scaler import GradScaler @@ -34,8 +34,13 @@ class OptimizerConfig(base_config.PrintableConfig): """Basic optimizer config with RAdam""" _target: Type = torch.optim.Adam + """The optimizer class to use.""" lr: float = 0.0005 + """The learning rate to use.""" eps: float = 1e-08 + """The epsilon value to use.""" + max_norm: Optional[float] = None + """The max norm to use for gradient clipping.""" # TODO: somehow make this more generic. i dont like the idea of overriding the setup function # but also not sure how to go about passing things into predefined torch objects. @@ -43,6 +48,7 @@ class OptimizerConfig(base_config.PrintableConfig): """Returns the instantiated object using the config.""" kwargs = vars(self).copy() kwargs.pop("_target") + kwargs.pop("max_norm") return self._target(params, **kwargs) @@ -52,6 +58,7 @@ class AdamOptimizerConfig(OptimizerConfig): _target: Type = torch.optim.Adam weight_decay: float = 0 + """The weight decay to use.""" @dataclass @@ -73,9 +80,11 @@ class Optimizers: self.config = config self.optimizers = {} self.schedulers = {} + self.parameters = {} for param_group_name, params in param_groups.items(): lr_init = config[param_group_name]["optimizer"].lr self.optimizers[param_group_name] = config[param_group_name]["optimizer"].setup(params=params) + self.parameters[param_group_name] = params if config[param_group_name]["scheduler"]: self.schedulers[param_group_name] = config[param_group_name]["scheduler"].setup( optimizer=self.optimizers[param_group_name], lr_init=lr_init @@ -109,13 +118,20 @@ class Optimizers: Args: grad_scaler: GradScaler to use """ - for _, optimizer in self.optimizers.items(): + for param_group, optimizer in self.optimizers.items(): + max_norm = self.config[param_group]["optimizer"].max_norm + if max_norm is not None: + grad_scaler.unscale_(optimizer) + torch.nn.utils.clip_grad_norm_(self.parameters[param_group], max_norm) grad_scaler.step(optimizer) def optimizer_step_all(self): """Run step for all optimizers.""" - for _, optimizer in self.optimizers.items(): + for param_group, optimizer in self.optimizers.items(): # note that they key is the parameter name + max_norm = self.config[param_group]["optimizer"].max_norm + if max_norm is not None: + torch.nn.utils.clip_grad_norm_(self.parameters[param_group], max_norm) optimizer.step() def scheduler_step_all(self, step: int) -> None: diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index f24ea5ccd..ac524e7b3 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -77,6 +77,9 @@ class TrainerConfig(ExperimentConfig): load_step: Optional[int] = None """Optionally specify model step to load from; if none, will find most recent model in load_dir.""" load_config: Optional[Path] = None + """Path to config YAML file.""" + log_gradients: bool = False + """Optionally log gradients during training""" class Trainer: @@ -188,7 +191,6 @@ class Trainer: step = 0 for step in range(self._start_step, self._start_step + num_iterations): with TimeWriter(writer, EventName.ITER_TRAIN_TIME, step=step) as train_t: - self.pipeline.train() # training callbacks before the training iteration @@ -290,7 +292,7 @@ class Trainer: @check_viewer_enabled def _update_viewer_rays_per_sec(self, train_t: TimeWriter, vis_t: TimeWriter, step: int): - """Performs update on rays/sec calclation for training + """Performs update on rays/sec calculation for training Args: train_t: timer object carrying time to execute total training iteration @@ -370,6 +372,18 @@ class Trainer: loss = functools.reduce(torch.add, loss_dict.values()) self.grad_scaler.scale(loss).backward() # type: ignore self.optimizers.optimizer_scaler_step_all(self.grad_scaler) + + if self.config.log_gradients: + total_grad = 0 + for tag, value in self.pipeline.model.named_parameters(): + assert tag != "Total" + if value.grad is not None: + grad = value.grad.norm() + metrics_dict[f"Gradients/{tag}"] = grad + total_grad += grad + + metrics_dict["Gradients/Total"] = total_grad + self.grad_scaler.update() self.optimizers.scheduler_step_all(step) diff --git a/nerfstudio/fields/instant_ngp_field.py b/nerfstudio/fields/instant_ngp_field.py index 18a806b68..846ea121e 100644 --- a/nerfstudio/fields/instant_ngp_field.py +++ b/nerfstudio/fields/instant_ngp_field.py @@ -58,7 +58,7 @@ class TCNNInstantNGPField(Field): num_layers_color: number of hidden layers for color network hidden_dim_color: dimension of hidden layers for color network use_appearance_embedding: whether to use appearance embedding - num_images: number of images, requried if use_appearance_embedding is True + num_images: number of images, required if use_appearance_embedding is True appearance_embedding_dim: dimension of appearance embedding contraction_type: type of contraction num_levels: number of levels of the hashmap for the base mlp diff --git a/nerfstudio/generative/__init__.py b/nerfstudio/generative/__init__.py new file mode 100644 index 000000000..d926cf95c --- /dev/null +++ b/nerfstudio/generative/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nerfstudio/generative/stable_diffusion.py b/nerfstudio/generative/stable_diffusion.py new file mode 100644 index 000000000..a91c3b49f --- /dev/null +++ b/nerfstudio/generative/stable_diffusion.py @@ -0,0 +1,354 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Stable Diffusion helpers""" + +# Modified from https://github.com/ashawkey/stable-dreamfusion/blob/main/nerf/sd.py + +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional, Tuple, Union + +import appdirs +import mediapy +import numpy as np +import torch +import torch.nn.functional as F +import tyro +from rich.console import Console +from torch import nn +from torchtyping import TensorType + +CONSOLE = Console(width=120) + +try: + from diffusers import PNDMScheduler, StableDiffusionPipeline + from transformers import logging + +except ImportError: + CONSOLE.print("[bold red]Missing Stable Diffusion packages.") + CONSOLE.print(r"Install using [yellow]pip install nerfstudio\[gen][/yellow]") + CONSOLE.print(r"or [yellow]pip install -e .\[gen][/yellow] if installing from source.") + sys.exit(1) + +logging.set_verbosity_error() +IMG_DIM = 512 +CONST_SCALE = 0.18215 + +SD_SOURCE = "runwayml/stable-diffusion-v1-5" +CLIP_SOURCE = "openai/clip-vit-large-patch14" + + +@dataclass +class UNet2DConditionOutput: + """Class to hold traced model""" + + sample: torch.FloatTensor + + +class StableDiffusion(nn.Module): + """Stable Diffusion implementation + Args: + device: device to use + num_train_timesteps: number of training timesteps + """ + + def __init__(self, device: Union[torch.device, str], num_train_timesteps: int = 1000) -> None: + super().__init__() + + self.device = device + self.num_train_timesteps = num_train_timesteps + + self.min_step = int(self.num_train_timesteps * 0.02) + self.max_step = int(self.num_train_timesteps * 0.98) + + self.scheduler = PNDMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + num_train_timesteps=self.num_train_timesteps, + ) + self.alphas = self.scheduler.alphas_cumprod.to(self.device) # type: ignore + + pipe = StableDiffusionPipeline.from_pretrained(SD_SOURCE, torch_dtype=torch.float16) + assert pipe is not None + pipe = pipe.to(self.device) + + pipe.enable_attention_slicing() + + # use jitted unet + unet_traced_filename = Path(appdirs.user_data_dir("nerfstudio")) / "sd_unet_traced.pt" + if unet_traced_filename.exists(): + CONSOLE.print("Loading traced UNet.") + unet_traced = torch.jit.load(unet_traced_filename) + + class TracedUNet(torch.nn.Module): + """Jitted version of UNet""" + + def __init__(self): + super().__init__() + self.in_channels = pipe.unet.in_channels + self.device = pipe.unet.device + + def forward(self, latent_model_input, t, encoder_hidden_states): # pylint: disable=no-self-use + """Forward pass""" + sample = unet_traced(latent_model_input, t, encoder_hidden_states)[0] + return UNet2DConditionOutput(sample=sample) + + self.unet = TracedUNet() + del pipe.unet + else: + CONSOLE.print("[bold yellow] Warning: Loading UNet without JIT acceleration.") + CONSOLE.print(r"Run [yellow]python scripts/generative/trace_stable_diffusion.py[/yellow] for a speedup!") + self.unet = pipe.unet + self.unet.to(memory_format=torch.channels_last) + + self.tokenizer = pipe.tokenizer + self.text_encoder = pipe.text_encoder + self.auto_encoder = pipe.vae + + CONSOLE.print("Stable Diffusion loaded!") + + def get_text_embeds( + self, prompt: Union[str, List[str]], negative_prompt: Union[str, List[str]] + ) -> TensorType[2, "max_length", "embed_dim"]: + """Get text embeddings for prompt and negative prompt + Args: + prompt: Prompt text + negative_prompt: Negative prompt text + Returns: + Text embeddings + """ + + # Tokenize text and get embeddings + text_input = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + + with torch.no_grad(): + text_embeddings = self.text_encoder(text_input.input_ids.to(self.device))[0] + + # Do the same for unconditional embeddings + uncond_input = self.tokenizer( + negative_prompt, padding="max_length", max_length=self.tokenizer.model_max_length, return_tensors="pt" + ) + + with torch.no_grad(): + uncond_embeddings = self.text_encoder(uncond_input.input_ids.to(self.device))[0] + + # Cat for final embeddings + text_embeddings = torch.cat([uncond_embeddings, text_embeddings]) + + return text_embeddings + + def sds_loss( + self, + text_embeddings: TensorType["N", "max_length", "embed_dim"], + image: TensorType["BS", 3, "H", "W"], + guidance_scale: float = 100.0, + ) -> Tuple[torch.Tensor, TensorType["BS", 4, "H", "W"], TensorType["BS", 4, "H", "W"]]: + """Score Distilation Sampling loss proposed in DreamFusion paper (https://dreamfusion3d.github.io/) + Args: + text_embeddings: Text embeddings + image: Rendered image + guidance_scale: How much to weigh the guidance + Returns: + A tuple of Loss, latent, and gradient + """ + image = F.interpolate(image, (IMG_DIM, IMG_DIM), mode="bilinear") + t = torch.randint(self.min_step, self.max_step + 1, [1], dtype=torch.long, device=self.device) + latents = self.imgs_to_latent(image) + + # predict the noise residual with unet, NO grad! + with torch.no_grad(): + # add noise + noise = torch.randn_like(latents) + latents_noisy = self.scheduler.add_noise(latents, noise, t) # type: ignore + # pred noise + latent_model_input = torch.cat([latents_noisy] * 2) + noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings).sample + + # perform guidance + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # w(t), sigma_t^2 + w = 1 - self.alphas[t] + + grad = w * (noise_pred - noise) + grad = torch.nan_to_num(grad) + + sds_mse = torch.mean(torch.nan_to_num(torch.square(noise_pred - noise))) + + return sds_mse, latents, grad + + def produce_latents( + self, + text_embeddings: TensorType["N", "max_length", "embed_dim"], + height: int = IMG_DIM, + width: int = IMG_DIM, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + latents: Optional[TensorType["BS", 4, "H", "W"]] = None, + ) -> TensorType["BS", 4, "H", "W"]: + """Produce latents for a given text embedding + Args: + text_embeddings: Text embeddings + height: Height of the image + width: Width of the image + num_inference_steps: Number of inference steps + guidance_scale: How much to weigh the guidance + latents: Latents to start with + Returns: + Latents + """ + + if latents is None: + latents = torch.randn( + (text_embeddings.shape[0] // 2, self.unet.in_channels, height // 8, width // 8), device=self.device + ) + + self.scheduler.set_timesteps(num_inference_steps) # type: ignore + + with torch.autocast("cuda"): + for t in self.scheduler.timesteps: # type: ignore + assert latents is not None + # expand the latents if we are doing classifier-free guidance to avoid doing two forward passes. + latent_model_input = torch.cat([latents] * 2) + + # predict the noise residual + with torch.no_grad(): + noise_pred = self.unet( + latent_model_input, t.to(self.device), encoder_hidden_states=text_embeddings + ).sample + + # perform guidance + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents)["prev_sample"] # type: ignore + return latents + + def latents_to_img(self, latents: TensorType["BS", 4, "H", "W"]) -> TensorType["BS", 3, "H", "W"]: + """Convert latents to images + Args: + latents: Latents to convert + Returns: + Images + """ + + latents = 1 / CONST_SCALE * latents + + with torch.no_grad(): + imgs = self.auto_encoder.decode(latents).sample + + imgs = (imgs / 2 + 0.5).clamp(0, 1) + + return imgs + + def imgs_to_latent(self, imgs: TensorType["BS", 3, "H", "W"]) -> TensorType["BS", 4, "H", "W"]: + """Convert images to latents + Args: + imgs: Images to convert + Returns: + Latents + """ + imgs = 2 * imgs - 1 + + posterior = self.auto_encoder.encode(imgs).latent_dist + latents = posterior.sample() * CONST_SCALE + + return latents + + def prompt_to_img( + self, + prompts: Union[str, List[str]], + negative_prompts: Union[str, List[str]] = "", + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + latents=None, + ) -> np.ndarray: + """Generate an images from a prompts. + Args: + prompts: The prompt to generate an image from. + negative_prompts: The negative prompt to generate an image from. + num_inference_steps: The number of inference steps to perform. + guidance_scale: The scale of the guidance. + latents: The latents to start from, defaults to random. + Returns: + The generated image. + """ + + prompts = [prompts] if isinstance(prompts, str) else prompts + negative_prompts = [negative_prompts] if isinstance(negative_prompts, str) else negative_prompts + text_embeddings = self.get_text_embeds(prompts, negative_prompts) + latents = self.produce_latents( + text_embeddings, + height=IMG_DIM, + width=IMG_DIM, + latents=latents, + num_inference_steps=num_inference_steps, + guidance_scale=guidance_scale, + ) # [1, 4, resolution, resolution] + + diffused_img = self.latents_to_img(latents.half()) + diffused_img = diffused_img.detach().cpu().permute(0, 2, 3, 1).numpy() + diffused_img = (diffused_img * 255).round().astype("uint8") + + return diffused_img + + def forward( + self, prompts, negative_prompts="", num_inference_steps=50, guidance_scale=7.5, latents=None + ) -> np.ndarray: + """Generate an image from a prompt. + Args: + prompts: The prompt to generate an image from. + negative_prompts: The negative prompt to generate an image from. + num_inference_steps: The number of inference steps to perform. + guidance_scale: The scale of the guidance. + latents: The latents to start from, defaults to random. + Returns: + The generated image. + """ + return self.prompt_to_img(prompts, negative_prompts, num_inference_steps, guidance_scale, latents) + + +def generate_image( + prompt: str, negative: str = "", seed: int = 0, steps: int = 50, save_path: Path = Path("test_sd.png") +): + """Generate an image from a prompt using Stable Diffusion. + Args: + prompt: The prompt to use. + negative: The negative prompt to use. + seed: The random seed to use. + steps: The number of steps to use. + save_path: The path to save the image to. + """ + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + cuda_device = torch.device("cuda") + with torch.no_grad(): + sd = StableDiffusion(cuda_device) + imgs = sd.prompt_to_img(prompt, negative, steps) + mediapy.write_image(str(save_path), imgs[0]) + + +if __name__ == "__main__": + tyro.cli(generate_image) diff --git a/nerfstudio/model_components/losses.py b/nerfstudio/model_components/losses.py index 7fcfe0419..353dd7db9 100644 --- a/nerfstudio/model_components/losses.py +++ b/nerfstudio/model_components/losses.py @@ -34,7 +34,7 @@ EPS = 1.0e-7 URF_SIGMA_SCALE_FACTOR = 3.0 -class DephtLossType(Enum): +class DepthLossType(Enum): """Types of depth losses for depth supervision.""" DS_NERF = 1 @@ -225,8 +225,11 @@ def ds_nerf_depth_loss( Returns: Depth loss scalar. """ + depth_mask = termination_depth > 0 + loss = -torch.log(weights + EPS) * torch.exp(-((steps - termination_depth[:, None]) ** 2) / (2 * sigma)) * lengths - return torch.mean(loss.sum(-2)) + loss = loss.sum(-2) * depth_mask + return torch.mean(loss) def urban_radiance_field_depth_loss( @@ -247,6 +250,8 @@ def urban_radiance_field_depth_loss( Returns: Depth loss scalar. """ + depth_mask = termination_depth > 0 + # Expected depth loss expected_depth_loss = (termination_depth - predicted_depth) ** 2 @@ -262,7 +267,8 @@ def urban_radiance_field_depth_loss( line_of_sight_loss_empty = (line_of_sight_loss_empty_mask * weights**2).sum(-2) line_of_sight_loss = line_of_sight_loss_near + line_of_sight_loss_empty - return torch.mean(expected_depth_loss + line_of_sight_loss) + loss = (expected_depth_loss + line_of_sight_loss) * depth_mask + return torch.mean(loss) def depth_loss( @@ -273,7 +279,7 @@ def depth_loss( sigma: TensorType[0], directions_norm: TensorType[..., 1], is_euclidean: bool, - depth_loss_type: DephtLossType, + depth_loss_type: DepthLossType, ) -> TensorType[0]: """Implementation of depth losses. @@ -291,14 +297,14 @@ def depth_loss( Depth loss scalar. """ if not is_euclidean: - termination_depth = termination_depth / directions_norm + termination_depth = termination_depth * directions_norm steps = (ray_samples.frustums.starts + ray_samples.frustums.ends) / 2 - if depth_loss_type == DephtLossType.DS_NERF: + if depth_loss_type == DepthLossType.DS_NERF: lengths = ray_samples.frustums.ends - ray_samples.frustums.starts return ds_nerf_depth_loss(weights, termination_depth, steps, lengths, sigma) - if depth_loss_type == DephtLossType.URF: + if depth_loss_type == DepthLossType.URF: return urban_radiance_field_depth_loss(weights, termination_depth, predicted_depth, steps, sigma) raise NotImplementedError("Provided depth loss type not implemented.") diff --git a/nerfstudio/model_components/renderers.py b/nerfstudio/model_components/renderers.py index 24ce46085..cf5af2a7d 100644 --- a/nerfstudio/model_components/renderers.py +++ b/nerfstudio/model_components/renderers.py @@ -37,7 +37,7 @@ from torchtyping import TensorType from typing_extensions import Literal from nerfstudio.cameras.rays import RaySamples -from nerfstudio.utils.math import components_from_spherical_harmonics +from nerfstudio.utils.math import components_from_spherical_harmonics, safe_normalize BACKGROUND_COLOR_OVERRIDE: Optional[TensorType[3]] = None @@ -55,7 +55,7 @@ def background_color_override_context(mode: TensorType[3]) -> Generator[None, No class RGBRenderer(nn.Module): - """Standard volumetic rendering. + """Standard volumetric rendering. Args: background_color: Background color as RGB. Uses random colors if None. @@ -165,7 +165,7 @@ class SHRenderer(nn.Module): """Composite samples along ray and render color image Args: - sh: Spherical hamonics coefficients for each sample + sh: Spherical harmonics coefficients for each sample directions: Sample direction weights: Weights for each sample @@ -328,7 +328,16 @@ class NormalsRenderer(nn.Module): cls, normals: TensorType["bs":..., "num_samples", 3], weights: TensorType["bs":..., "num_samples", 1], + normalize: bool = True, ) -> TensorType["bs":..., 3]: - """Calculate normals along the ray.""" + """Calculate normals along the ray. + + Args: + normals: Normals for each sample. + weights: Weights of each sample. + normalize: Normalize normals. + """ n = torch.sum(weights * normals, dim=-2) + if normalize: + n = safe_normalize(n) return n diff --git a/nerfstudio/model_components/shaders.py b/nerfstudio/model_components/shaders.py new file mode 100644 index 000000000..e71388e27 --- /dev/null +++ b/nerfstudio/model_components/shaders.py @@ -0,0 +1,77 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Shaders for rendering.""" +from typing import Optional + +from torch import nn +from torchtyping import TensorType + + +class LambertianShader(nn.Module): + """Calculate Lambertian shading.""" + + @classmethod + def forward( + cls, + rgb: TensorType["bs":..., 3], + normals: TensorType["bs":..., 3], + light_direction: TensorType["bs":..., 3], + shading_weight: float = 1.0, + detach_normals=True, + ): + """Calculate Lambertian shading. + + Args: + rgb: Accumulated rgb along a ray. + normals: Accumulated normals along a ray. + light_direction: Direction of light source. + shading_weight: Lambertian shading (1.0) vs. ambient lighting (0.0) ratio + detach_normals: Detach normals from the computation graph when computing shading. + + Returns: + Textureless Lambertian shading, Lambertian shading + """ + if detach_normals: + normals = normals.detach() + + lambertian = (1 - shading_weight) + shading_weight * (normals @ light_direction).clamp(min=0) + shaded = lambertian.unsqueeze(-1).repeat(1, 3) + shaded_albedo = rgb * lambertian.unsqueeze(-1) + + return shaded, shaded_albedo + + +class NormalsShader(nn.Module): + """Calculate shading for normals.""" + + @classmethod + def forward( + cls, + normals: TensorType["bs":..., 3], + weights: Optional[TensorType["bs":..., 1]] = None, + ): + """Applies a rainbow colormap to the normals. + + Args: + normals: Normalized 3D vectors. + weights: Optional weights to scale to the normal colors. (Can be used for masking) + + Returns: + Colored normals + """ + normals = (normals + 1) / 2 + if weights is not None: + normals = normals * weights + return normals diff --git a/nerfstudio/models/depth_nerfacto.py b/nerfstudio/models/depth_nerfacto.py index da4c59002..2bea84c7b 100644 --- a/nerfstudio/models/depth_nerfacto.py +++ b/nerfstudio/models/depth_nerfacto.py @@ -24,7 +24,7 @@ from typing import Dict, Tuple, Type import torch from nerfstudio.cameras.rays import RayBundle -from nerfstudio.model_components.losses import DephtLossType, depth_loss +from nerfstudio.model_components.losses import DepthLossType, depth_loss from nerfstudio.models.nerfacto import NerfactoModel, NerfactoModelConfig from nerfstudio.utils import colormaps @@ -46,7 +46,7 @@ class DepthNerfactoModelConfig(NerfactoModelConfig): """Starting uncertainty around depth values in meters (defaults to 0.2m).""" sigma_decay_rate: float = 0.99985 """Rate of exponential decay.""" - depth_loss_type: DephtLossType = DephtLossType.DS_NERF + depth_loss_type: DepthLossType = DepthLossType.DS_NERF """Depth loss type.""" @@ -108,7 +108,7 @@ class DepthNerfactoModel(NerfactoModel): metrics, images = super().get_image_metrics_and_images(outputs, batch) ground_truth_depth = batch["depth_image"] if not self.config.is_euclidean_depth: - ground_truth_depth = ground_truth_depth / outputs["directions_norm"] + ground_truth_depth = ground_truth_depth * outputs["directions_norm"] ground_truth_depth_colormap = colormaps.apply_depth_colormap(ground_truth_depth) predicted_depth_colormap = colormaps.apply_depth_colormap( @@ -118,7 +118,10 @@ class DepthNerfactoModel(NerfactoModel): far_plane=torch.max(ground_truth_depth), ) images["depth"] = torch.cat([ground_truth_depth_colormap, predicted_depth_colormap], dim=1) - metrics["depth_mse"] = torch.nn.functional.mse_loss(outputs["depth"], ground_truth_depth) + depth_mask = ground_truth_depth > 0 + metrics["depth_mse"] = torch.nn.functional.mse_loss( + outputs["depth"][depth_mask], ground_truth_depth[depth_mask] + ) return metrics, images def _get_sigma(self): diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py index c996e20b3..7442bfe8b 100644 --- a/nerfstudio/models/nerfacto.py +++ b/nerfstudio/models/nerfacto.py @@ -54,6 +54,7 @@ from nerfstudio.model_components.renderers import ( RGBRenderer, ) from nerfstudio.model_components.scene_colliders import NearFarCollider +from nerfstudio.model_components.shaders import NormalsShader from nerfstudio.models.base_model import Model, ModelConfig from nerfstudio.utils import colormaps @@ -187,6 +188,9 @@ class NerfactoModel(Model): self.renderer_depth = DepthRenderer() self.renderer_normals = NormalsRenderer() + # shaders + self.normals_shader = NormalsShader() + # losses self.rgb_loss = MSELoss() @@ -250,9 +254,10 @@ class NerfactoModel(Model): } if self.config.predict_normals: - outputs["normals"] = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights) - outputs["pred_normals"] = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights) - + normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights) + pred_normals = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights) + outputs["normals"] = self.normals_shader(normals) + outputs["pred_normals"] = self.normals_shader(pred_normals) # These use a lot of GPU memory, so we avoid storing them for eval. if self.training: outputs["weights_list"] = weights_list @@ -334,12 +339,6 @@ class NerfactoModel(Model): images_dict = {"img": combined_rgb, "accumulation": combined_acc, "depth": combined_depth} - # normals to RGB for visualization. TODO: use a colormap - if "normals" in outputs: - images_dict["normals"] = (outputs["normals"] + 1.0) / 2.0 - if "pred_normals" in outputs: - images_dict["pred_normals"] = (outputs["pred_normals"] + 1.0) / 2.0 - for i in range(self.config.num_proposal_iterations): key = f"prop_depth_{i}" prop_depth_i = colormaps.apply_depth_colormap( diff --git a/nerfstudio/plugins/__init__.py b/nerfstudio/plugins/__init__.py new file mode 100644 index 000000000..d926cf95c --- /dev/null +++ b/nerfstudio/plugins/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nerfstudio/plugins/registry.py b/nerfstudio/plugins/registry.py new file mode 100644 index 000000000..365f05a12 --- /dev/null +++ b/nerfstudio/plugins/registry.py @@ -0,0 +1,50 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Module that keeps all registered plugins and allows for plugin discovery. +""" + +import sys +import typing as t + +from rich.progress import Console + +from nerfstudio.plugins.types import MethodSpecification + +if sys.version_info < (3, 10): + from importlib_metadata import entry_points +else: + from importlib.metadata import entry_points +CONSOLE = Console(width=120) + + +def discover_methods(): + """ + Discovers all methods registered using the `nerfstudio.method_configs` entrypoint. + """ + methods = {} + descriptions = {} + discovered_entry_points = entry_points(group="nerfstudio.method_configs") + for name in discovered_entry_points.names: + specification = discovered_entry_points[name].load() + if not isinstance(specification, MethodSpecification): + CONSOLE.print( + "[bold yellow]Warning: Could not entry point {n} as it is not an instance of MethodSpecification" + ) + continue + specification = t.cast(MethodSpecification, specification) + methods[specification.config.method_name] = specification.config + descriptions[specification.config.method_name] = specification.description + return methods, descriptions diff --git a/nerfstudio/plugins/types.py b/nerfstudio/plugins/types.py new file mode 100644 index 000000000..ee7e8e470 --- /dev/null +++ b/nerfstudio/plugins/types.py @@ -0,0 +1,33 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This package contains specifications used to register plugins. +""" +from dataclasses import dataclass + +from nerfstudio.engine.trainer import TrainerConfig + + +@dataclass +class MethodSpecification: + """ + Method specification class used to register custom methods with Nerfstudio. + The registered methods will be available in commands such as `ns-train` + """ + + config: TrainerConfig + """Trainer configuration""" + description: str + """Method description shown in `ns-train` help""" diff --git a/nerfstudio/process_data/colmap_utils.py b/nerfstudio/process_data/colmap_utils.py index 9db20647f..93f47adfa 100644 --- a/nerfstudio/process_data/colmap_utils.py +++ b/nerfstudio/process_data/colmap_utils.py @@ -604,6 +604,14 @@ def colmap_to_json( cameras = read_cameras_binary(cameras_path) images = read_images_binary(images_path) + # Images were renamed to frame_{i:05d}.{ext} and + # the filenames needs to be replaced in the transforms.json as well + original_filenames = [x.name for x in images.values()] + # Sort was used in nerfstudio.process_data.process_data_utils:get_image_filenames + original_filenames.sort() + # Build the map to the new filenames + filename_map = {name: f"frame_{i+1:05d}{os.path.splitext(name)[-1]}" for i, name in enumerate(original_filenames)} + # Only supports one camera camera_params = cameras[1].params @@ -619,7 +627,8 @@ def colmap_to_json( c2w = c2w[np.array([1, 0, 2, 3]), :] c2w[2, :] *= -1 - name = Path(f"./images/{im_data.name}") + name = filename_map[im_data.name] + name = Path(f"./images/{name}") frame = { "file_path": name.as_posix(), diff --git a/nerfstudio/process_data/metashape_utils.py b/nerfstudio/process_data/metashape_utils.py index 491c06a2c..17d7771e8 100644 --- a/nerfstudio/process_data/metashape_utils.py +++ b/nerfstudio/process_data/metashape_utils.py @@ -58,10 +58,14 @@ def metashape_to_json( # pylint: disable=too-many-statements sensors = chunk.find("sensors") # TODO Add support for per-frame intrinsics - if sensors is None or len(sensors) != 1: + if sensors is None: + raise ValueError("No sensors found") + + calibrated_sensors = [sensor for sensor in sensors if sensor.find("calibration")] + if len(calibrated_sensors) != 1: raise ValueError("Only one sensor is supported for now") - sensor = sensors.find("sensor") + sensor = calibrated_sensors[0] data = {} @@ -98,6 +102,14 @@ def metashape_to_json( # pylint: disable=too-many-statements if camera_label not in image_filename_map: continue frame["file_path"] = image_filename_map[camera_label].as_posix() + + if camera.get("sensor_id") != sensor.get("id"): + # this should only happen when we have a sensor that doesn't have calibration + if verbose: + CONSOLE.print(f"Missing sensor calibration for {camera.get('label')}, Skipping") + num_skipped += 1 + continue + if camera.find("transform") is None: if verbose: CONSOLE.print(f"Missing transforms data for {camera.get('label')}, Skipping") diff --git a/nerfstudio/process_data/polycam_utils.py b/nerfstudio/process_data/polycam_utils.py index d1887833b..079ff16f7 100644 --- a/nerfstudio/process_data/polycam_utils.py +++ b/nerfstudio/process_data/polycam_utils.py @@ -72,7 +72,7 @@ def polycam_to_json( frame["h"] = frame_json["height"] - crop_border_pixels * 2 frame["file_path"] = f"./images/frame_{i+1:05d}{image_filename.suffix}" if use_depth: - frame["depth_map_path"] = f"./depth/frame_{i+1:05d}{depth_filenames[i].suffix}" + frame["depth_file_path"] = f"./depth/frame_{i+1:05d}{depth_filenames[i].suffix}" # Transform matrix to nerfstudio format. Please refer to the documentation for coordinate system conventions. frame["transform_matrix"] = [ [frame_json["t_20"], frame_json["t_21"], frame_json["t_22"], frame_json["t_23"]], diff --git a/nerfstudio/process_data/process_data_utils.py b/nerfstudio/process_data/process_data_utils.py index d8d9206e9..4919edd93 100644 --- a/nerfstudio/process_data/process_data_utils.py +++ b/nerfstudio/process_data/process_data_utils.py @@ -46,6 +46,19 @@ CAMERA_MODELS = { } +def list_images(data: Path) -> List[Path]: + """Lists all supported images in a directory + + Args: + data: Path to the directory of images. + Returns: + Paths to images contained in the directory + """ + allowed_exts = [".jpg", ".jpeg", ".png", ".tif", ".tiff"] + image_paths = sorted([p for p in data.glob("[!.]*") if p.suffix.lower() in allowed_exts]) + return image_paths + + def get_image_filenames(directory: Path, max_num_images: int = -1) -> Tuple[List[Path], int]: """Returns a list of image filenames in a directory. @@ -55,8 +68,7 @@ def get_image_filenames(directory: Path, max_num_images: int = -1) -> Tuple[List Returns: A tuple of A list of image filenames, number of original image paths. """ - allowed_exts = [".jpg", ".jpeg", ".png", ".tif", ".tiff"] - image_paths = sorted([p for p in directory.glob("[!.]*") if p.suffix.lower() in allowed_exts]) + image_paths = list_images(directory) num_orig_images = len(image_paths) if max_num_images != -1 and num_orig_images > max_num_images: @@ -78,8 +90,8 @@ def get_num_frames_in_video(video: Path) -> int: Returns: The number of frames in a video. """ - cmd = f"ffprobe -v error -select_streams v:0 -count_packets \ - -show_entries stream=nb_read_packets -of csv=p=0 {video}" + cmd = f'ffprobe -v error -select_streams v:0 -count_packets \ + -show_entries stream=nb_read_packets -of csv=p=0 "{video}"' output = run_command(cmd) assert output is not None output = output.strip(" ,\t\n\r") @@ -114,7 +126,7 @@ def convert_video_to_images( print("Number of frames in video:", num_frames) out_filename = image_dir / "frame_%05d.png" - ffmpeg_cmd = f"ffmpeg -i {video_path}" + ffmpeg_cmd = f'ffmpeg -i "{video_path}"' spacing = num_frames // num_frames_target if spacing > 1: @@ -168,7 +180,7 @@ def copy_images_list( file_type = image_paths[0].suffix filename = f"frame_%05d{file_type}" crop = f"crop=iw-{crop_border_pixels*2}:ih-{crop_border_pixels*2}" - ffmpeg_cmd = f"ffmpeg -y -noautorotate -i {image_dir / filename} -q:v 2 -vf {crop} {image_dir / filename}" + ffmpeg_cmd = f'ffmpeg -y -noautorotate -i "{image_dir / filename}" -q:v 2 -vf {crop} "{image_dir / filename}"' run_command(ffmpeg_cmd, verbose=verbose) num_frames = len(image_paths) @@ -210,9 +222,9 @@ def copy_and_upscale_polycam_depth_maps_list( for idx, depth_map in enumerate(polycam_depth_image_filenames): destination = depth_dir / f"frame_{idx + 1:05d}{depth_map.suffix}" ffmpeg_cmd = [ - f"ffmpeg -y -i {depth_map} ", + f'ffmpeg -y -i "{depth_map}" ', f"-q:v 2 -vf scale=iw*{upscale_factor}:ih*{upscale_factor}:flags=neighbor ", - f"{destination}", + f'"{destination}"', ] ffmpeg_cmd = " ".join(ffmpeg_cmd) run_command(ffmpeg_cmd, verbose=verbose) @@ -222,7 +234,7 @@ def copy_and_upscale_polycam_depth_maps_list( file_type = depth_dir.glob("frame_*").__next__().suffix filename = f"frame_%05d{file_type}" crop = f"crop=iw-{crop_border_pixels * 2}:ih-{crop_border_pixels * 2}" - ffmpeg_cmd = f"ffmpeg -y -i {depth_dir / filename} -q:v 2 -vf {crop} {depth_dir / filename}" + ffmpeg_cmd = f'ffmpeg -y -i "{depth_dir / filename}" -q:v 2 -vf {crop} "{depth_dir / filename}"' run_command(ffmpeg_cmd, verbose=verbose) CONSOLE.log("[bold green]:tada: Done upscaling depth maps.") @@ -240,8 +252,7 @@ def copy_images(data: Path, image_dir: Path, verbose) -> int: The number of images copied. """ with status(msg="[bold yellow]Copying images...", spinner="bouncingBall", verbose=verbose): - allowed_exts = [".jpg", ".jpeg", ".png", ".tif", ".tiff"] - image_paths = sorted([p for p in data.glob("[!.]*") if p.suffix.lower() in allowed_exts]) + image_paths = list_images(data) if len(image_paths) == 0: CONSOLE.log("[bold red]:skull: No usable images in the data folder.") @@ -282,9 +293,9 @@ def downscale_images(image_dir: Path, num_downscales: int, folder_name: str = "i for f in files: filename = f.name ffmpeg_cmd = [ - f"ffmpeg -y -noautorotate -i {image_dir / filename} ", + f'ffmpeg -y -noautorotate -i "{image_dir / filename}" ', f"-q:v 2 -vf scale=iw/{downscale_factor}:ih/{downscale_factor} ", - f"{downscale_dir / filename}", + f'"{downscale_dir / filename}"', ] ffmpeg_cmd = " ".join(ffmpeg_cmd) run_command(ffmpeg_cmd, verbose=verbose) diff --git a/nerfstudio/utils/install_checks.py b/nerfstudio/utils/install_checks.py index 23351a1b2..05e505d4e 100644 --- a/nerfstudio/utils/install_checks.py +++ b/nerfstudio/utils/install_checks.py @@ -39,3 +39,11 @@ def check_colmap_installed(): CONSOLE.print("[bold red]Could not find COLMAP. Please install COLMAP.") print("See https://colmap.github.io/install.html for installation instructions.") sys.exit(1) + + +def check_curl_installed(): + """Checks if curl is installed.""" + curl_path = shutil.which("curl") + if curl_path is None: + CONSOLE.print("[bold red]Could not find [yellow]curl[red], Please install [yellow]curl") + sys.exit(1) diff --git a/nerfstudio/utils/math.py b/nerfstudio/utils/math.py index 2658b1c39..cee6108a9 100644 --- a/nerfstudio/utils/math.py +++ b/nerfstudio/utils/math.py @@ -15,10 +15,13 @@ """ Math Helper Functions """ from dataclasses import dataclass +from typing import Tuple import torch from torchtyping import TensorType +_USE_NERFACC = True + def components_from_spherical_harmonics(levels: int, directions: TensorType[..., 3]) -> TensorType[..., "components"]: """ @@ -103,7 +106,7 @@ def compute_3d_gaussian( dir_variance: TensorType[..., 1], radius_variance: TensorType[..., 1], ) -> Gaussians: - """Compute guassian along ray. + """Compute gaussian along ray. Args: directions: Axis of Gaussian. @@ -193,47 +196,92 @@ def expected_sin(x_means: torch.Tensor, x_vars: torch.Tensor) -> torch.Tensor: return torch.exp(-0.5 * x_vars) * torch.sin(x_means) -def intersect_aabb(origins, directions, aabb): +@torch.jit.script +def _intersect_aabb( + origins: torch.Tensor, + directions: torch.Tensor, + aabb: torch.Tensor, + max_bound: float = 1e10, + invalid_value: float = 1e10, +) -> Tuple[torch.Tensor, torch.Tensor]: """ - pytorch implementation of ray intersection with AABB box - :param origin: N,3 tensor of 3d positions - :param direction: N,3 tensor of normalized directions - :param aabb: 6,1 array of aabb box in the form of [x_min, y_min, z_min, x_max, y_max, z_max] - :return: t_min, t_max - two tensors of shapes N,1 representing the intersection signed distance from - the origin. t_min is clipped to 0 in case it is negative, in case of no intersection t_min is 1e10. + Implementation of ray intersection with AABB box + + Args: + origins: [N,3] tensor of 3d positions + directions: [N,3] tensor of normalized directions + aabb: [6] array of aabb box in the form of [x_min, y_min, z_min, x_max, y_max, z_max] + max_bound: Maximum value of t_max + invalid_value: Value to return in case of no intersection + + Returns: + t_min, t_max - two tensors of shapes N representing distance of intersection from the origin. """ - max_bound = 1e10 + tx_min = (aabb[:3] - origins) / directions + tx_max = (aabb[3:] - origins) / directions - tx_min = (aabb[0] - origins[:, 0]) / directions[:, 0] - tx_max = (aabb[3] - origins[:, 0]) / directions[:, 0] t_min = torch.min(tx_min, tx_max) t_max = torch.max(tx_min, tx_max) - ty_min_temp = (aabb[1] - origins[:, 1]) / directions[:, 1] - ty_max_temp = (aabb[4] - origins[:, 1]) / directions[:, 1] - ty_min = torch.min(ty_min_temp, ty_max_temp) - ty_max = torch.max(ty_min_temp, ty_max_temp) + t_min = torch.max(t_min, dim=-1).values + t_max = torch.min(t_max, dim=-1).values - cond = torch.logical_or((t_min > ty_max), (ty_min > t_max)) - t_min = torch.where(cond, max_bound, t_min) - t_max = torch.where(cond, max_bound, t_max) + t_min = torch.clamp(t_min, min=0, max=max_bound) + t_max = torch.clamp(t_max, min=0, max=max_bound) - t_min = torch.max(t_min, ty_min) - t_max = torch.min(t_max, ty_max) - - tz_min_temp = (aabb[2] - origins[:, 2]) / directions[:, 2] - tz_max_temp = (aabb[5] - origins[:, 2]) / directions[:, 2] - tz_min = torch.min(tz_min_temp, tz_max_temp) - tz_max = torch.max(tz_min_temp, tz_max_temp) - - cond = torch.logical_or((t_min > tz_max), tz_min > t_max) - t_min = torch.where(cond, max_bound, t_min) - t_max = torch.where(cond, max_bound, t_max) - - t_min = torch.max(t_min, tz_min) - t_max = torch.min(t_max, tz_max) - t_min = torch.where(t_min > 0, t_min, 0) - t_max = torch.where(t_min == max_bound, max_bound, t_max) + cond = t_max <= t_min + t_min = torch.where(cond, invalid_value, t_min) + t_max = torch.where(cond, invalid_value, t_max) return t_min, t_max + + +def intersect_aabb( + origins: TensorType["N", 3], + directions: TensorType["N", 3], + aabb: TensorType[6], +) -> Tuple[TensorType["N"], TensorType["N"]]: + """ + Implementation of ray intersection with AABB box + + Args: + origins: 3d positions + directions: Normalized directions + aabb: array of aabb box in the form of [x_min, y_min, z_min, x_max, y_max, z_max] + max_bound: Maximum value of t_max + invalid_value: Value to return in case of no intersection + + Returns: + t_min, t_max - two tensors of shapes N representing distance of intersection from the origin. + """ + + global _USE_NERFACC # pylint: disable=global-statement + if _USE_NERFACC: + try: + import nerfacc # pylint: disable=import-outside-toplevel + + t_min, t_max = nerfacc.ray_aabb_intersect(origins, directions, aabb) + except: # pylint: disable=bare-except + t_min, t_max = _intersect_aabb(origins, directions, aabb, max_bound=1e10, invalid_value=1e10) + _USE_NERFACC = False + else: + t_min, t_max = _intersect_aabb(origins, directions, aabb, max_bound=1e10, invalid_value=1e10) + + return t_min, t_max + + +def safe_normalize( + vectors: TensorType["batch_dim":..., "N"], + eps: float = 1e-10, +) -> TensorType["batch_dim":..., "N"]: + """Normalizes vectors. + + Args: + vectors: Vectors to normalize. + eps: Epsilon value to avoid division by zero. + + Returns: + Normalized vectors. + """ + return vectors / (torch.norm(vectors, dim=-1, keepdim=True) + eps) diff --git a/nerfstudio/viewer/app/package.json b/nerfstudio/viewer/app/package.json index 37f2b7bbb..171ec8dd1 100644 --- a/nerfstudio/viewer/app/package.json +++ b/nerfstudio/viewer/app/package.json @@ -1,7 +1,7 @@ { "name": "viewer", "homepage": ".", - "version": "23-01-25-0", + "version": "23-02-3-0", "private": true, "dependencies": { "@emotion/react": "^11.10.4", diff --git a/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx b/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx index a1c331fad..66b92458e 100644 --- a/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx @@ -37,6 +37,12 @@ export function RenderControls() { const colormapChoice = useSelector( (state) => state.renderingState.colormap_choice, ); + const colormapInvert = useSelector( + (state) => state.renderingState.colormap_invert, + ); + const colormapNormalize = useSelector( + (state) => state.renderingState.colormap_normalize, + ); const max_resolution = useSelector( (state) => state.renderingState.maxResolution, ); @@ -115,6 +121,51 @@ export function RenderControls() { }, disabled: colormapOptions.length === 1, }, + colormap_invert: { + label: '| Invert', + value: colormapInvert, + hint: 'Invert the colormap', + onChange: (v) => { + dispatch_and_send( + websocket, + dispatch, + 'renderingState/colormap_invert', + v, + ); + }, + render: (get) => get('colormap_options') !== 'default', + }, + colormap_normalize: { + label: '| Normalize', + value: colormapNormalize, + hint: 'Whether to normalize output between 0 and 1', + onChange: (v) => { + dispatch_and_send( + websocket, + dispatch, + 'renderingState/colormap_normalize', + v, + ); + }, + render: (get) => get('colormap_options') !== 'default', + }, + colormap_range: { + label: '| Range', + value: [0, 1], + step: 0.01, + min: -2, + max: 5, + hint: 'Min and max values of the colormap', + onChange: (v) => { + dispatch_and_send( + websocket, + dispatch, + 'renderingState/colormap_range', + v, + ); + }, + render: (get) => get('colormap_options') !== 'default', + }, // Dynamic Resolution target_train_util: { label: 'Train Util.', diff --git a/nerfstudio/viewer/app/src/reducer.js b/nerfstudio/viewer/app/src/reducer.js index 6de1ea7c2..d8335c80d 100644 --- a/nerfstudio/viewer/app/src/reducer.js +++ b/nerfstudio/viewer/app/src/reducer.js @@ -30,12 +30,17 @@ const initialState = { all_camera_paths: null, // object containing camera paths and names - isTraining: true, + + // colormap options output_options: ['rgb'], // populated by the possible Graph outputs output_choice: 'rgb', // the selected output colormap_options: ['default'], // populated by the output choice colormap_choice: 'default', // the selected colormap + colormap_invert: false, // whether to invert the colormap + colormap_normalize: false, // whether to normalize the colormap + colormap_range: [0.0, 1.0], // the range of the colormap + maxResolution: 1024, targetTrainUtil: 0.9, eval_res: '?', @@ -51,7 +56,7 @@ const initialState = { // Crop Box Options crop_enabled: false, - crop_bg_color: {r: 38, g:42, b:55}, + crop_bg_color: { r: 38, g: 42, b: 55 }, crop_scale: [2.0, 2.0, 2.0], crop_center: [0.0, 0.0, 0.0], }, diff --git a/nerfstudio/viewer/server/utils.py b/nerfstudio/viewer/server/utils.py index b0763b6d1..a21b02d82 100644 --- a/nerfstudio/viewer/server/utils.py +++ b/nerfstudio/viewer/server/utils.py @@ -63,7 +63,7 @@ def three_js_perspective_camera_focal_length(fov: float, image_height: int): def get_intrinsics_matrix_and_camera_to_world_h( - camera_object: Dict[str, Any], image_height: int + camera_object: Dict[str, Any], image_height: int, image_width: Optional[int] = None ) -> Tuple[torch.Tensor, torch.Tensor]: """Returns the camera intrinsics matrix and the camera to world homogeneous matrix. @@ -74,7 +74,8 @@ def get_intrinsics_matrix_and_camera_to_world_h( # intrinsics fov = camera_object["fov"] aspect = camera_object["aspect"] - image_width = aspect * image_height + if image_width is None: + image_width = aspect * image_height pp_w = image_width / 2.0 pp_h = image_height / 2.0 if (camera_object["camera_type"] == "perspective") | (camera_object["camera_type"] == "fisheye"): diff --git a/nerfstudio/viewer/server/viewer_utils.py b/nerfstudio/viewer/server/viewer_utils.py index 44056537f..2c1541467 100644 --- a/nerfstudio/viewer/server/viewer_utils.py +++ b/nerfstudio/viewer/server/viewer_utils.py @@ -48,7 +48,6 @@ from nerfstudio.models.base_model import Model from nerfstudio.utils import colormaps, profiler, writer from nerfstudio.utils.decorators import check_main_thread, decorate_all from nerfstudio.utils.io import load_from_json, write_to_json -from nerfstudio.utils.misc import get_dict_to_torch from nerfstudio.utils.writer import GLOBAL_BUFFER, EventName, TimeWriter from nerfstudio.viewer.server.subprocess import run_viewer_bridge_server_as_subprocess from nerfstudio.viewer.server.utils import ( @@ -83,7 +82,7 @@ def setup_viewer(config: cfg.ViewerConfig, log_filename: Path, datapath: str): class OutputTypes(str, enum.Enum): - """Noncomprehsnive list of output render types""" + """Noncomprehensive list of output render types""" INIT = "init" RGB = "rgb" @@ -93,14 +92,14 @@ class OutputTypes(str, enum.Enum): class ColormapTypes(str, enum.Enum): - """Noncomprehsnive list of colormap render types""" + """List of colormap render types""" - INIT = "init" DEFAULT = "default" TURBO = "turbo" - DEPTH = "depth" - SEMANTIC = "semantic" - BOOLEAN = "boolean" + VIRIDIS = "viridis" + MAGMA = "magma" + INFERNO = "inferno" + CIVIDIS = "cividis" class IOChangeException(Exception): @@ -163,7 +162,6 @@ class RenderThread(threading.Thread): self.exc = e if outputs: - outputs = get_dict_to_torch(outputs) self.vis_outputs = outputs self.state.check_done_render = True @@ -222,12 +220,15 @@ class CheckThread(threading.Thread): # check colormap type colormap_type = self.state.vis["renderingState/colormap_choice"].read() - if colormap_type is None: - colormap_type = ColormapTypes.INIT if self.state.prev_colormap_type != colormap_type: self.state.check_interrupt_vis = True return + colormap_range = self.state.vis["renderingState/colormap_range"].read() + if self.state.prev_colormap_range != colormap_range: + self.state.check_interrupt_vis = True + return + # check max render max_resolution = self.state.vis["renderingState/maxResolution"].read() if max_resolution is not None: @@ -287,7 +288,10 @@ class ViewerState: self.prev_camera_matrix = None self.prev_render_time = 0 self.prev_output_type = OutputTypes.INIT - self.prev_colormap_type = ColormapTypes.INIT + self.prev_colormap_type = None + self.prev_colormap_invert = False + self.prev_colormap_normalize = False + self.prev_colormap_range = [0, 1] self.prev_moving = False self.output_type_changed = True self.max_resolution = 1000 @@ -585,11 +589,21 @@ class ViewerState: self.camera_moving = True colormap_type = self.vis["renderingState/colormap_choice"].read() - if colormap_type is None: - colormap_type = ColormapTypes.INIT if self.prev_colormap_type != colormap_type: self.camera_moving = True + colormap_range = self.vis["renderingState/colormap_range"].read() + if self.prev_colormap_range != colormap_range: + self.camera_moving = True + + colormap_invert = self.vis["renderingState/colormap_invert"].read() + if self.prev_colormap_invert != colormap_invert: + self.camera_moving = True + + colormap_normalize = self.vis["renderingState/colormap_normalize"].read() + if self.prev_colormap_normalize != colormap_normalize: + self.camera_moving = True + crop_bg_color = self.vis["renderingState/crop_bg_color"].read() if self.prev_crop_enabled: if self.prev_crop_bg_color != crop_bg_color: @@ -623,37 +637,28 @@ class ViewerState: return outputs[reformatted_output] # rendering depth outputs - if self.prev_colormap_type == ColormapTypes.DEPTH or ( - self.prev_colormap_type == ColormapTypes.DEFAULT - and outputs[reformatted_output].dtype == torch.float - and (torch.max(outputs[reformatted_output]) - 1.0) > eps # handle floating point arithmetic - ): - accumulation_str = ( - OutputTypes.ACCUMULATION - if OutputTypes.ACCUMULATION in self.output_list - else OutputTypes.ACCUMULATION_FINE - ) - return colormaps.apply_depth_colormap(outputs[reformatted_output], accumulation=outputs[accumulation_str]) - - # rendering accumulation outputs - if self.prev_colormap_type == ColormapTypes.TURBO or ( - self.prev_colormap_type == ColormapTypes.DEFAULT and outputs[reformatted_output].dtype == torch.float - ): - return colormaps.apply_colormap(outputs[reformatted_output]) + if outputs[reformatted_output].shape[-1] == 1 and outputs[reformatted_output].dtype == torch.float: + output = outputs[reformatted_output] + if self.prev_colormap_normalize: + output = output - torch.min(output) + output = output / (torch.max(output) + eps) + output = output * (self.prev_colormap_range[1] - self.prev_colormap_range[0]) + self.prev_colormap_range[0] + output = torch.clip(output, 0, 1) + if self.prev_colormap_invert: + output = 1 - output + if self.prev_colormap_type == ColormapTypes.DEFAULT: + return colormaps.apply_colormap(output, cmap=ColormapTypes.TURBO.value) + return colormaps.apply_colormap(output, cmap=self.prev_colormap_type) # rendering semantic outputs - if self.prev_colormap_type == ColormapTypes.SEMANTIC or ( - self.prev_colormap_type == ColormapTypes.DEFAULT and outputs[reformatted_output].dtype == torch.int - ): + if outputs[reformatted_output].dtype == torch.int: logits = outputs[reformatted_output] labels = torch.argmax(torch.nn.functional.softmax(logits, dim=-1), dim=-1) # type: ignore assert colors is not None return colors[labels] # rendering boolean outputs - if self.prev_colormap_type == ColormapTypes.BOOLEAN or ( - self.prev_colormap_type == ColormapTypes.DEFAULT and outputs[reformatted_output].dtype == torch.bool - ): + if outputs[reformatted_output].dtype == torch.bool: return colormaps.apply_boolean_colormap(outputs[reformatted_output]) raise NotImplementedError @@ -664,34 +669,38 @@ class ViewerState: # returns the description to for WebRTC to the specific websocket connection offer = RTCSessionDescription(data["sdp"], data["type"]) - if self.config.skip_openrelay: - ice_servers = [ - RTCIceServer(urls="stun:stun.l.google.com:19302"), - ] + if self.config.local: + pc = RTCPeerConnection() else: - ice_servers = [ - RTCIceServer(urls="stun:stun.l.google.com:19302"), - RTCIceServer(urls="stun:openrelay.metered.ca:80"), - RTCIceServer( - urls="turn:openrelay.metered.ca:80", username="openrelayproject", credential="openrelayproject" - ), - RTCIceServer( - urls="turn:openrelay.metered.ca:443", username="openrelayproject", credential="openrelayproject" - ), - RTCIceServer( - urls="turn:openrelay.metered.ca:443?transport=tcp", - username="openrelayproject", - credential="openrelayproject", - ), - ] + if self.config.skip_openrelay: + ice_servers = [ + RTCIceServer(urls="stun:stun.l.google.com:19302"), + ] + else: + ice_servers = [ + RTCIceServer(urls="stun:stun.l.google.com:19302"), + RTCIceServer(urls="stun:openrelay.metered.ca:80"), + RTCIceServer( + urls="turn:openrelay.metered.ca:80", username="openrelayproject", credential="openrelayproject" + ), + RTCIceServer( + urls="turn:openrelay.metered.ca:443", username="openrelayproject", credential="openrelayproject" + ), + RTCIceServer( + urls="turn:openrelay.metered.ca:443?transport=tcp", + username="openrelayproject", + credential="openrelayproject", + ), + ] - pc = RTCPeerConnection(configuration=RTCConfiguration(iceServers=ice_servers)) + pc = RTCPeerConnection(configuration=RTCConfiguration(iceServers=ice_servers)) self.pcs.add(pc) video = SingleFrameStreamTrack() self.video_tracks.add(video) video_sender = pc.addTrack(video) - force_codec(pc, video_sender, "video/VP8") + print(f"viewer using video/{self.config.codec}") + force_codec(pc, video_sender, f"video/{self.config.codec}") await pc.setRemoteDescription(offer) answer = await pc.createAnswer() @@ -714,13 +723,12 @@ class ViewerState: for video_track in self.video_tracks: video_track.put_frame(image) - def _send_output_to_viewer(self, outputs: Dict[str, Any], colors: torch.Tensor = None, eps=1e-6): + def _send_output_to_viewer(self, outputs: Dict[str, Any], colors: torch.Tensor = None): """Chooses the correct output and sends it to the viewer Args: outputs: the dictionary of outputs to choose from, from the graph colors: is only set if colormap is for semantics. Defaults to None. - eps: epsilon to handle floating point comparisons """ if self.output_list is None: self.output_list = list(outputs.keys()) @@ -733,16 +741,13 @@ class ViewerState: reformatted_output = self._process_invalid_output(self.prev_output_type) # re-register colormaps and send to viewer - if self.output_type_changed or self.prev_colormap_type == ColormapTypes.INIT: + if self.output_type_changed or self.prev_colormap_type is None: self.prev_colormap_type = ColormapTypes.DEFAULT - colormap_options = [ColormapTypes.DEFAULT] - if ( - outputs[reformatted_output].shape[-1] != 3 - and outputs[reformatted_output].dtype == torch.float - and (torch.max(outputs[reformatted_output]) - 1.0) <= eps # handle floating point arithmetic - ): - # accumulation can also include depth - colormap_options.extend(["depth"]) + colormap_options = [] + if outputs[reformatted_output].shape[-1] == 3: + colormap_options = [ColormapTypes.DEFAULT] + if outputs[reformatted_output].shape[-1] == 1 and outputs[reformatted_output].dtype == torch.float: + colormap_options = list(ColormapTypes) self.output_type_changed = False self.vis["renderingState/colormap_choice"].write(self.prev_colormap_type) self.vis["renderingState/colormap_options"].write(colormap_options) @@ -828,11 +833,16 @@ class ViewerState: else: image_height = (num_vis_rays / aspect_ratio) ** 0.5 image_height = int(round(image_height, -1)) - image_height = min(self.max_resolution, image_height) + image_height = max(min(self.max_resolution, image_height), 30) image_width = int(image_height * aspect_ratio) if image_width > self.max_resolution: image_width = self.max_resolution image_height = int(image_width / aspect_ratio) + if self.config.codec != "VP8": + # force even values to allow hardware encoder usage + quantize = 2 + image_width = int(image_width / quantize) * quantize + image_height = int(image_height / quantize) * quantize return image_height, image_width def _process_invalid_output(self, output_type: str) -> str: @@ -881,9 +891,17 @@ class ViewerState: # check and perform colormap type updates colormap_type = self.vis["renderingState/colormap_choice"].read() - colormap_type = ColormapTypes.INIT if colormap_type is None else colormap_type self.prev_colormap_type = colormap_type + colormap_invert = self.vis["renderingState/colormap_invert"].read() + self.prev_colormap_invert = colormap_invert + + colormap_normalize = self.vis["renderingState/colormap_normalize"].read() + self.prev_colormap_normalize = colormap_normalize + + colormap_range = self.vis["renderingState/colormap_range"].read() + self.prev_colormap_range = colormap_range + # update render aabb try: self._update_render_aabb(graph) @@ -906,7 +924,7 @@ class ViewerState: return intrinsics_matrix, camera_to_world_h = get_intrinsics_matrix_and_camera_to_world_h( - camera_object, image_height=image_height + camera_object, image_height=image_height, image_width=image_width ) camera_to_world = camera_to_world_h[:3, :] diff --git a/pyproject.toml b/pyproject.toml index 10942cf21..ba57c01d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,18 +25,19 @@ dependencies = [ "functorch==0.2.1", "h5py>=2.9.0", "imageio>=2.21.1", + 'importlib-metadata>=6.0.0; python_version < "3.10"', "ipywidgets>=7.6", "jupyterlab>=3.3.4", "matplotlib>=3.5.3", "mediapy>=1.1.0", "msgpack>=1.0.4", "msgpack_numpy>=0.4.8", - "nerfacc==0.3.3", + "nerfacc==0.3.4", "open3d>=0.16.0", "opencv-python==4.6.0.66", "Pillow>=9.3.0", "plotly>=5.7.0", - "protobuf<=3.20.3", + "protobuf<=3.20.3,!=3.20.0", "pymeshlab>=2022.2.post2", "pyngrok>=5.1.0", "python-socketio>=5.7.1", @@ -59,6 +60,14 @@ dependencies = [ [project.optional-dependencies] + +# Generative related dependencies +gen = [ + "diffusers==0.9.0", + "transformers==4.24.0", +] + + # Development packages dev = [ "black[jupyter]==22.3.0", diff --git a/scripts/downloads/download_data.py b/scripts/downloads/download_data.py index 12a0a0227..89c77391f 100755 --- a/scripts/downloads/download_data.py +++ b/scripts/downloads/download_data.py @@ -15,6 +15,7 @@ from rich.console import Console from typing_extensions import Annotated from nerfstudio.configs.base_config import PrintableConfig +from nerfstudio.utils import install_checks from nerfstudio.utils.scripts import run_command CONSOLE = Console(width=120) @@ -212,6 +213,7 @@ class DNerfDownload(DatasetDownload): """Download the D-NeRF dataset (https://github.com/albertpumarola/D-NeRF).""" # TODO: give this code the same structure as download_nerfstudio + install_checks.check_curl_installed() final_path = save_dir / Path("dnerf") if os.path.exists(final_path): shutil.rmtree(str(final_path)) @@ -253,6 +255,7 @@ class PhototourismDownload(DatasetDownload): def download(self, save_dir: Path): """Download a PhotoTourism dataset: https://www.cs.ubc.ca/~kmyi/imw2020/data.html""" + install_checks.check_curl_installed() if self.capture_name == "all": for capture_name in phototourism_downloads: if capture_name != "all": diff --git a/scripts/generative/__init__.py b/scripts/generative/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/generative/trace_stable_diffusion.py b/scripts/generative/trace_stable_diffusion.py new file mode 100644 index 000000000..a501bd424 --- /dev/null +++ b/scripts/generative/trace_stable_diffusion.py @@ -0,0 +1,74 @@ +"""Trace Stable Diffusion UNet for speed improvement""" + +import functools +import sys +from pathlib import Path + +import appdirs +import torch +import tyro +from rich.console import Console + +CONSOLE = Console(width=120) + +try: + from diffusers import StableDiffusionPipeline +except ImportError: + CONSOLE.print("[bold red]Missing Stable Diffusion packages.") + CONSOLE.print(r"Install using [yellow]pip install nerfstudio\[gen][/yellow]") + CONSOLE.print(r"or [yellow]pip install -e .\[gen][/yellow] if installing from source.") + sys.exit(1) + + +def jit_unet(save_dir: Path = Path(appdirs.user_data_dir("nerfstudio"))): + """Trace Stable Diffusion UNet for speed improvement + + Args: + save_dir: directory to save the traced model + """ + + # torch disable grad + torch.set_grad_enabled(False) + + # load inputs + def generate_inputs(): + sample = torch.randn(2, 4, 64, 64).half().cuda() + timestep = torch.rand(1).half().cuda() * 999 + encoder_hidden_states = torch.randn(2, 77, 768).half().cuda() + return sample, timestep, encoder_hidden_states + + pipe = StableDiffusionPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", + torch_dtype=torch.float16, + ).to("cuda") + + pipe.enable_attention_slicing() + + unet = pipe.unet + unet.eval() + unet.to(memory_format=torch.channels_last) # use channels_last memory format + unet.forward = functools.partial(unet.forward, return_dict=False) # set return_dict=False as default + + # warmup + inputs = None + for _ in range(3): + with torch.inference_mode(): + inputs = generate_inputs() + _ = unet(*inputs) + + # trace + print("tracing..") + assert inputs is not None + unet_traced = torch.jit.trace(unet, inputs) + unet_traced.eval() + print("done tracing") + + # save the traced model + unet_traced_filename = save_dir / "sd_unet_traced.pt" + if not unet_traced_filename.exists(): + unet_traced_filename.parent.mkdir(parents=True, exist_ok=True) + unet_traced.save(unet_traced_filename) + + +if __name__ == "__main__": + tyro.cli(jit_unet) diff --git a/scripts/process_data.py b/scripts/process_data.py index 4727e2b2c..f6f9a9a7f 100755 --- a/scripts/process_data.py +++ b/scripts/process_data.py @@ -28,6 +28,7 @@ from nerfstudio.process_data.process_data_utils import CAMERA_MODELS from nerfstudio.utils import install_checks CONSOLE = Console(width=120) +DEFAULT_COLMAP_PATH = Path("colmap/sparse/0") @dataclass @@ -74,6 +75,12 @@ class ProcessImages: will downscale the images by 2x, 4x, and 8x.""" skip_colmap: bool = False """If True, skips COLMAP and generates transforms.json if possible.""" + skip_image_processing: bool = False + """If True, skips copying and downscaling of images and only runs COLMAP if possible and enabled""" + colmap_model_path: Path = DEFAULT_COLMAP_PATH + """Optionally sets the path of the colmap model. Used only when --skip-colmap is set to True. + The path is relative to the output directory. + """ colmap_cmd: str = "colmap" """How to call the COLMAP executable.""" gpu: bool = True @@ -83,6 +90,19 @@ class ProcessImages: def main(self) -> None: """Process images into a nerfstudio dataset.""" + require_cameras_exist = False + colmap_model_path = self.output_dir / Path(self.colmap_model_path) + if self.colmap_model_path != DEFAULT_COLMAP_PATH: + if not self.skip_colmap: + CONSOLE.log("[bold red]The --colmap-model-path can only be used when --skip-colmap is not set.") + sys.exit(1) + elif not (self.output_dir / self.colmap_model_path).exists(): + CONSOLE.log( + f"[bold red]The colmap-model-path {self.output_dir / self.colmap_model_path} does not exist." + ) + sys.exit(1) + require_cameras_exist = True + install_checks.check_ffmpeg_installed() install_checks.check_colmap_installed() @@ -92,57 +112,46 @@ class ProcessImages: summary_log = [] - # Copy images to output directory - num_frames = process_data_utils.copy_images(self.data, image_dir=image_dir, verbose=self.verbose) - summary_log.append(f"Starting with {num_frames} images") + # Copy and downscale images + if not self.skip_image_processing: + # Copy images to output directory + num_frames = process_data_utils.copy_images(self.data, image_dir=image_dir, verbose=self.verbose) + summary_log.append(f"Starting with {num_frames} images") - # Downscale images - summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose)) + # Downscale images + summary_log.append( + process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose) + ) + else: + num_frames = len(process_data_utils.list_images(self.data)) + if num_frames == 0: + CONSOLE.log("[bold red]:skull: No usable images in the data folder.") + sys.exit(1) + summary_log.append(f"Starting with {num_frames} images") # Run COLMAP colmap_dir = self.output_dir / "colmap" if not self.skip_colmap: colmap_dir.mkdir(parents=True, exist_ok=True) + colmap_model_path = colmap_dir / "sparse" / "0" + require_cameras_exist = True - (sfm_tool, feature_type, matcher_type) = process_data_utils.find_tool_feature_matcher_combination( - self.sfm_tool, self.feature_type, self.matcher_type - ) - - if sfm_tool == "colmap": - colmap_utils.run_colmap( - image_dir=image_dir, - colmap_dir=colmap_dir, - camera_model=CAMERA_MODELS[self.camera_type], - gpu=self.gpu, - verbose=self.verbose, - matching_method=self.matching_method, - colmap_cmd=self.colmap_cmd, - ) - elif sfm_tool == "hloc": - hloc_utils.run_hloc( - image_dir=image_dir, - colmap_dir=colmap_dir, - camera_model=CAMERA_MODELS[self.camera_type], - verbose=self.verbose, - matching_method=self.matching_method, - feature_type=feature_type, - matcher_type=matcher_type, - ) - else: - CONSOLE.log("[bold red]Invalid combination of sfm_tool, feature_type, and matcher_type, exiting") - sys.exit(1) + self._run_colmap(image_dir, colmap_dir) # Save transforms.json - if (colmap_dir / "sparse" / "0" / "cameras.bin").exists(): + if (colmap_model_path / "cameras.bin").exists(): with CONSOLE.status("[bold yellow]Saving results to transforms.json", spinner="balloon"): num_matched_frames = colmap_utils.colmap_to_json( - cameras_path=colmap_dir / "sparse" / "0" / "cameras.bin", - images_path=colmap_dir / "sparse" / "0" / "images.bin", + cameras_path=colmap_model_path / "cameras.bin", + images_path=colmap_model_path / "images.bin", output_dir=self.output_dir, camera_model=CAMERA_MODELS[self.camera_type], ) summary_log.append(f"Colmap matched {num_matched_frames} images") summary_log.append(colmap_utils.get_matching_summary(num_frames, num_matched_frames)) + elif require_cameras_exist: + CONSOLE.log(f"[bold red]Could not find existing COLMAP results ({colmap_model_path / 'cameras.bin'}).") + sys.exit(1) else: CONSOLE.log("[bold yellow]Warning: could not find existing COLMAP results. Not generating transforms.json") @@ -152,6 +161,35 @@ class ProcessImages: CONSOLE.print(summary, justify="center") CONSOLE.rule() + def _run_colmap(self, image_dir, colmap_dir): + (sfm_tool, feature_type, matcher_type) = process_data_utils.find_tool_feature_matcher_combination( + self.sfm_tool, self.feature_type, self.matcher_type + ) + + if sfm_tool == "colmap": + colmap_utils.run_colmap( + image_dir=image_dir, + colmap_dir=colmap_dir, + camera_model=CAMERA_MODELS[self.camera_type], + gpu=self.gpu, + verbose=self.verbose, + matching_method=self.matching_method, + colmap_cmd=self.colmap_cmd, + ) + elif sfm_tool == "hloc": + hloc_utils.run_hloc( + image_dir=image_dir, + colmap_dir=colmap_dir, + camera_model=CAMERA_MODELS[self.camera_type], + verbose=self.verbose, + matching_method=self.matching_method, + feature_type=feature_type, + matcher_type=matcher_type, + ) + else: + CONSOLE.log("[bold red]Invalid combination of sfm_tool, feature_type, and matcher_type, exiting") + sys.exit(1) + @dataclass class ProcessVideo: @@ -350,7 +388,7 @@ class ProcessInsta360: if not filename_back.exists(): raise FileNotFoundError(f"Could not find {filename_back}") - ffprobe_cmd = f"ffprobe -v quiet -print_format json -show_streams -select_streams v:0 {filename_back}" + ffprobe_cmd = f'ffprobe -v quiet -print_format json -show_streams -select_streams v:0 "{filename_back}"' ffprobe_output = process_data_utils.run_command(ffprobe_cmd) diff --git a/scripts/train.py b/scripts/train.py index 240fdeca0..f19b0ccc5 100755 --- a/scripts/train.py +++ b/scripts/train.py @@ -152,7 +152,7 @@ def launch( config: Optional[TrainerConfig] = None, timeout: timedelta = DEFAULT_TIMEOUT, ) -> None: - """Function that spawns muliple processes to call on main_func + """Function that spawns multiple processes to call on main_func Args: main_func (Callable): function that will be called by the distributed workers diff --git a/tests/plugins/test_registry.py b/tests/plugins/test_registry.py new file mode 100644 index 000000000..fe8361a18 --- /dev/null +++ b/tests/plugins/test_registry.py @@ -0,0 +1,51 @@ +""" +Tests for the nerfstudio.plugins.registry module. +""" +import sys + +from nerfstudio.engine.trainer import TrainerConfig +from nerfstudio.pipelines.base_pipeline import VanillaPipelineConfig +from nerfstudio.plugins import registry +from nerfstudio.plugins.types import MethodSpecification + +if sys.version_info < (3, 10): + import importlib_metadata +else: + from importlib import metadata as importlib_metadata + + +TestConfig = MethodSpecification( + config=TrainerConfig( + method_name="test-method", + pipeline=VanillaPipelineConfig(), + optimizers={}, + ), + description="Test description", +) + + +def test_discover_methods(): + """Tests if a custom method gets properly registered using the discover_methods method""" + entry_points_backup = registry.entry_points + + def entry_points(group=None): + return importlib_metadata.EntryPoints( + [ + importlib_metadata.EntryPoint( + name="test", value="test_registry:TestConfig", group="nerfstudio.method_configs" + ) + ] + ).select(group=group) + + try: + # Mock importlib entry_points + registry.entry_points = entry_points + + # Discover plugins + methods, _ = registry.discover_methods() + assert "test-method" in methods + config = methods["test-method"] + assert isinstance(config, TrainerConfig) + finally: + # Revert mock + registry.entry_points = entry_points_backup