From 78eeb3bfe8389623cb08837f4f647d0ad6ee5444 Mon Sep 17 00:00:00 2001 From: Pablo Vela Date: Tue, 14 Feb 2023 18:57:55 -0600 Subject: [PATCH 01/17] devcontainer for VSCode development (#1428) add devcontainer for easy development in VSCode --- .devcontainer/devcontainer.json | 13 +++++++++++++ .devcontainer/docker-compose.yml | 28 ++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/docker-compose.yml diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..84f99bf49 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,13 @@ +{ + "name": "NerfStudio Dev Container", + "dockerComposeFile": "docker-compose.yml", + "service": "nerfstudio", + "workspaceFolder": "/workspace", + "shutdownAction": "stopCompose", + "extensions": [ + "mutantdino.resourcemonitor", + "ms-azuretools.vscode-docker", + "nvidia.nsight-vscode-edition", + "ms-python.python" + ] +} \ No newline at end of file diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 000000000..6376422cf --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,28 @@ +version: "3.8" + +services: + nerfstudio: + shm_size: '12gb' + image: dromni/nerfstudio:0.1.17 + # build: + # context: .. + # dockerfile: ./Dockerfile + stdin_open: true + tty: true + environment: + NVIDIA_DRIVER_CAPABILITIES: compute,utility,graphics + DISPLAY: $DISPLAY + ports: + - "7007:7007" + volumes: + - ../:/workspace/ + - /tmp/.X11-unix:/tmp/.X11-unix + working_dir: /workspace/ + command: /bin/bash + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [ gpu ] \ No newline at end of file From 97498e3223ac13c219e32b0c6305fbcfbf2cb77e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=C3=A1=C5=A1=20Kulh=C3=A1nek?= Date: Wed, 15 Feb 2023 17:46:55 +0100 Subject: [PATCH 02/17] Improve ns-install-cli (#1396) * Improve ns-install-cli * Remove unused code * ns-install-cli conda specify encoding explicitly * Fix missing directory in ns-install-cli * Implement ns-install-cli deactivate for conda env * ns-install-cli: conda create deactivate.d --- scripts/completions/install.py | 283 +++++++++++++++++++++++---------- 1 file changed, 199 insertions(+), 84 deletions(-) diff --git a/scripts/completions/install.py b/scripts/completions/install.py index 636ed1979..98e67fea2 100755 --- a/scripts/completions/install.py +++ b/scripts/completions/install.py @@ -10,27 +10,29 @@ import shutil import stat import subprocess import sys -from typing import List, Union +from typing import List, Optional, Union import tyro from rich.console import Console from rich.prompt import Confirm from typing_extensions import Literal, assert_never +from typing_extensions import get_args as typing_get_args + +if sys.version_info < (3, 10): + import importlib_metadata +else: + from importlib import metadata as importlib_metadata ConfigureMode = Literal["install", "uninstall"] ShellType = Literal["zsh", "bash"] CONSOLE = Console(width=120) +HEADER_LINE = "# Source nerfstudio autocompletions." -ENTRYPOINTS = [ - "ns-install-cli", - "ns-process-data", - "ns-download-data", - "ns-train", - "ns-eval", - "ns-render", - "ns-dev-test", -] + +def _get_all_entry_points() -> List[str]: + entry_points = importlib_metadata.distribution("nerfstudio").entry_points + return [x.name for x in entry_points] def _check_tyro_cli(script_path: pathlib.Path) -> bool: @@ -127,6 +129,62 @@ def _exclamation() -> str: return random.choice(["Cool", "Nice", "Neat", "Great", "Exciting", "Excellent", "Ok"]) + "!" +def _get_deactivate_script(commands: List[str], shell: Optional[ShellType], add_header=True) -> str: + if shell is None: + # Install the universal script + result_script = [] + for shell_type in typing_get_args(ShellType): + result_script.append(f'if [ -n "${shell_type.upper()}_VERSION" ]; then') + result_script.append(_get_deactivate_script(commands, shell_type, add_header=False)) + result_script.append("fi") + source_lines = "\n".join(result_script) + + elif shell == "zsh": + source_lines = "\n".join([f"unset '_comps[{command}]' &> /dev/null" for command in commands]) + elif shell == "bash": + source_lines = "\n".join([f"complete -r {command} &> /dev/null" for command in commands]) + else: + assert_never(shell) + + if add_header: + source_lines = f"\n{HEADER_LINE}\n{source_lines}" + return source_lines + + +def _get_source_script(completions_dir: pathlib.Path, shell: Optional[ShellType], add_header=True) -> str: + if shell is None: + # Install the universal script + result_script = [] + for shell_type in typing_get_args(ShellType): + result_script.append(f'if [ -n "${shell_type.upper()}_VERSION" ]; then') + result_script.append(_get_source_script(completions_dir, shell_type, add_header=False)) + result_script.append("fi") + source_lines = "\n".join(result_script) + + elif shell == "zsh": + source_lines = "\n".join( + [ + "if ! command -v compdef &> /dev/null; then", + " autoload -Uz compinit", + " compinit", + "fi", + f"source {completions_dir / 'setup.zsh'}", + ] + ) + elif shell == "bash": + source_lines = "\n".join( + [ + f"source {completions_dir / 'setup.bash'}", + ] + ) + else: + assert_never(shell) + + if add_header: + source_lines = f"\n{HEADER_LINE}\n{source_lines}" + return source_lines + + def _update_rc( completions_dir: pathlib.Path, mode: ConfigureMode, @@ -141,36 +199,13 @@ def _update_rc( """ # Install or uninstall `source_line`. - header_line = "# Source nerfstudio autocompletions." - if shell == "zsh": - source_lines = "\n".join( - [ - "", - header_line, - "if ! command -v compdef &> /dev/null; then", - " autoload -Uz compinit", - " compinit", - "fi", - f"source {completions_dir / 'setup.zsh'}", - ] - ) - elif shell == "bash": - source_lines = "\n".join( - [ - "", - header_line, - f"source {completions_dir / 'setup.bash'}", - ] - ) - else: - assert_never(shell) - + source_lines = _get_source_script(completions_dir, shell) rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc" # Always try to uninstall previous completions. rc_source = rc_path.read_text() - while header_line in rc_source: - before_install, _, after_install = rc_source.partition(header_line) + while HEADER_LINE in rc_source: + before_install, _, after_install = rc_source.partition(HEADER_LINE) source_file, _, after_install = after_install.partition("\nsource ")[2].partition("\n") assert source_file.endswith(f"/completions/setup.{shell}") rc_source = before_install + after_install @@ -189,6 +224,108 @@ def _update_rc( assert mode == "uninstall" +def _update_conda_scripts( + commands: List[str], + completions_dir: pathlib.Path, + mode: ConfigureMode, +) -> None: + """Try to add a `source /.../completions/setup.{shell}` line automatically to conda's activation scripts. + + Args: + completions_dir: Path to location of this script. + mode: Install or uninstall completions. + """ + + # Install or uninstall `source_line`. + activate_source_lines = _get_source_script(completions_dir, None) + deactivate_source_lines = _get_deactivate_script(commands, None) + + conda_path = pathlib.Path(os.environ["CONDA_PREFIX"]) + activate_path = conda_path / "etc/conda/activate.d/nerfstudio_activate.sh" + deactivate_path = conda_path / "etc/conda/deactivate.d/nerfstudio_deactivate.sh" + if mode == "uninstall": + if activate_path.exists(): + os.remove(activate_path) + if deactivate_path.exists(): + os.remove(deactivate_path) + CONSOLE.log(f":broom: Existing completions uninstalled from {conda_path}.") + elif mode == "install": + # Install completions. + activate_path.parent.mkdir(exist_ok=True, parents=True) + deactivate_path.parent.mkdir(exist_ok=True, parents=True) + with activate_path.open("w+", encoding="utf8") as f: + f.write(activate_source_lines) + with deactivate_path.open("w+", encoding="utf8") as f: + f.write(deactivate_source_lines) + CONSOLE.log( + f":person_gesturing_ok: Completions installed to {conda_path}. {_exclamation()} Reactivate the environment" + " to try them out." + ) + else: + assert_never(mode) + + +def _get_conda_path() -> Optional[pathlib.Path]: + """ + Returns the path to the conda environment if + the nerfstudio package is installed in one. + """ + conda_path = None + if "CONDA_PREFIX" in os.environ: + # Conda is active, we will check if the Nerfstudio is installed in the conda env. + distribution = importlib_metadata.distribution("nerfstudio") + if str(distribution.locate_file("nerfstudio")).startswith(os.environ["CONDA_PREFIX"]): + conda_path = pathlib.Path(os.environ["CONDA_PREFIX"]) + return conda_path + + +def _generate_completions_files( + completions_dir: pathlib.Path, + scripts_dir: pathlib.Path, + shells_supported: List[ShellType], + shells_found: List[ShellType], +) -> None: + # Set to True to install completions for scripts as well. + include_scripts = False + + # Find tyro CLIs. + script_paths = list(filter(_check_tyro_cli, scripts_dir.glob("**/*.py"))) if include_scripts else [] + script_names = tuple(p.name for p in script_paths) + assert len(set(script_names)) == len(script_names) + + # Get existing completion files. + existing_completions = set() + for shell in shells_supported: + target_dir = completions_dir / shell + if target_dir.exists(): + existing_completions |= set(target_dir.glob("*")) + + # Get all entry_points. + entry_points = _get_all_entry_points() + + # Run generation jobs. + concurrent_executor = concurrent.futures.ThreadPoolExecutor() + with CONSOLE.status("[bold]:writing_hand: Generating completions...", spinner="bouncingBall"): + completion_paths = list( + concurrent_executor.map( + lambda path_or_entrypoint_and_shell: _generate_completion( + path_or_entrypoint_and_shell[0], path_or_entrypoint_and_shell[1], completions_dir + ), + itertools.product(script_paths + entry_points, shells_found), + ) + ) + + # Delete obsolete completion files. + for unexpected_path in set(p.absolute() for p in existing_completions) - set( + p.absolute() for p in completion_paths + ): + if unexpected_path.is_dir(): + shutil.rmtree(unexpected_path) + elif unexpected_path.exists(): + unexpected_path.unlink() + CONSOLE.log(f":broom: Deleted {unexpected_path}.") + + def main(mode: ConfigureMode = "install") -> None: """Main script. @@ -201,16 +338,24 @@ def main(mode: ConfigureMode = "install") -> None: CONSOLE.log("[bold red]$HOME is not set. Exiting.") return + # Get conda path if in conda environment. + conda_path = _get_conda_path() + # Try to locate the user's bashrc or zshrc. - shells_supported: List[ShellType] = ["zsh", "bash"] - shells_found: List[ShellType] = [] - for shell in shells_supported: - rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc" - if not rc_path.exists(): - CONSOLE.log(f":person_shrugging: {rc_path.name} not found, skipping.") - else: - CONSOLE.log(f":mag: Found {rc_path.name}!") - shells_found.append(shell) + shells_supported: List[ShellType] = list(typing_get_args(ShellType)) + if conda_path is not None: + # Running in conda; we have to support all shells. + shells_found = shells_supported + CONSOLE.log(f":mag: Detected conda environment {conda_path}!") + else: + shells_found: List[ShellType] = [] + for shell in shells_supported: + rc_path = pathlib.Path(os.environ["HOME"]) / f".{shell}rc" + if not rc_path.exists(): + CONSOLE.log(f":person_shrugging: {rc_path.name} not found, skipping.") + else: + CONSOLE.log(f":mag: Found {rc_path.name}!") + shells_found.append(shell) # Get scripts/ directory. completions_dir = pathlib.Path(__file__).absolute().parent @@ -230,48 +375,18 @@ def main(mode: ConfigureMode = "install") -> None: else: CONSOLE.log(f":heavy_check_mark: No existing completions at: {target_dir}.") elif mode == "install": - # Set to True to install completions for scripts as well. - include_scripts = False - - # Find tyro CLIs. - script_paths = list(filter(_check_tyro_cli, scripts_dir.glob("**/*.py"))) if include_scripts else [] - script_names = tuple(p.name for p in script_paths) - assert len(set(script_names)) == len(script_names) - - # Get existing completion files. - existing_completions = set() - for shell in shells_supported: - target_dir = completions_dir / shell - if target_dir.exists(): - existing_completions |= set(target_dir.glob("*")) - - # Run generation jobs. - concurrent_executor = concurrent.futures.ThreadPoolExecutor() - with CONSOLE.status("[bold]:writing_hand: Generating completions...", spinner="bouncingBall"): - completion_paths = list( - concurrent_executor.map( - lambda path_or_entrypoint_and_shell: _generate_completion( - path_or_entrypoint_and_shell[0], path_or_entrypoint_and_shell[1], completions_dir - ), - itertools.product(script_paths + ENTRYPOINTS, shells_found), - ) - ) - - # Delete obsolete completion files. - for unexpected_path in set(p.absolute() for p in existing_completions) - set( - p.absolute() for p in completion_paths - ): - if unexpected_path.is_dir(): - shutil.rmtree(unexpected_path) - elif unexpected_path.exists(): - unexpected_path.unlink() - CONSOLE.log(f":broom: Deleted {unexpected_path}.") + _generate_completions_files(completions_dir, scripts_dir, shells_supported, shells_found) else: assert_never(mode) - # Install or uninstall from bashrc/zshrc. - for shell in shells_found: - _update_rc(completions_dir, mode, shell) + if conda_path is not None: + # In conda environment we add the completitions activation scripts. + commands = _get_all_entry_points() + _update_conda_scripts(commands, completions_dir, mode) + else: + # Install or uninstall from bashrc/zshrc. + for shell in shells_found: + _update_rc(completions_dir, mode, shell) CONSOLE.print("[bold]All done![/bold]") From be5274db098c5a91146313c3497b918e5b0ce3a3 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Wed, 15 Feb 2023 10:05:18 -0800 Subject: [PATCH 03/17] Add arxiv link (#1434) --- README.md | 2 ++ docs/index.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/README.md b/README.md index 7ea21deb8..0b0d8592c 100644 --- a/README.md +++ b/README.md @@ -302,6 +302,8 @@ We provide the following support structures to make life easier for getting star # Citation +You can find a paper writeup of the framework on [arXiv](https://arxiv.org/abs/2302.04264). + If you use this library or find the documentation useful for your research, please consider citing: ``` diff --git a/docs/index.md b/docs/index.md index 023b87cc8..88e4c90ae 100644 --- a/docs/index.md +++ b/docs/index.md @@ -155,6 +155,8 @@ We'll be constantly growing this list! So make sure to check back in to see our ## Citation +You can find a paper writeup of the framework on [arXiv](https://arxiv.org/abs/2302.04264). + If you use this library or find the documentation useful for your research, please consider citing: ```none From 6619dd1ac2e01f42e391c9045aecea9fd64522ff Mon Sep 17 00:00:00 2001 From: Pablo Vela Date: Wed, 15 Feb 2023 13:49:35 -0600 Subject: [PATCH 04/17] add monosdf normal loss and midas scale invariant depth loss (#1410) * add monosdf normal loss and midas scale invariant depth loss * add context to scale/shift calc and normal loss, consolidate MidasMSE loss * consolidated, add info on gradient reg term * add info on ssi * more documentation and type annotation * move compute scale/shift to class * fix pylint errors * fix tensortype error * fix comments * change single var name and update comment * fix input tensortype * fix black error * fix mask shape --- nerfstudio/model_components/losses.py | 217 ++++++++++++++++++++++++++ nerfstudio/utils/math.py | 26 +++ 2 files changed, 243 insertions(+) diff --git a/nerfstudio/model_components/losses.py b/nerfstudio/model_components/losses.py index 353dd7db9..419a8a1f4 100644 --- a/nerfstudio/model_components/losses.py +++ b/nerfstudio/model_components/losses.py @@ -20,8 +20,10 @@ from enum import Enum import torch from torch import nn from torchtyping import TensorType +from typing_extensions import Literal from nerfstudio.cameras.rays import RaySamples +from nerfstudio.utils.math import masked_reduction L1Loss = nn.L1Loss MSELoss = nn.MSELoss @@ -308,3 +310,218 @@ def depth_loss( return urban_radiance_field_depth_loss(weights, termination_depth, predicted_depth, steps, sigma) raise NotImplementedError("Provided depth loss type not implemented.") + + +def monosdf_normal_loss( + normal_pred: TensorType["num_samples", 3], normal_gt: TensorType["num_samples", 3] +) -> TensorType[0]: + """ + Normal consistency loss proposed in monosdf - https://niujinshuchong.github.io/monosdf/ + Enforces consistency between the volume rendered normal and the predicted monocular normal. + With both angluar and L1 loss. Eq 14 https://arxiv.org/pdf/2206.00665.pdf + Args: + normal_pred: volume rendered normal + normal_gt: monocular normal + """ + normal_gt = torch.nn.functional.normalize(normal_gt, p=2, dim=-1) + normal_pred = torch.nn.functional.normalize(normal_pred, p=2, dim=-1) + l1 = torch.abs(normal_pred - normal_gt).sum(dim=-1).mean() + cos = (1.0 - torch.sum(normal_pred * normal_gt, dim=-1)).mean() + return l1 + cos + + +class MiDaSMSELoss(nn.Module): + """ + data term from MiDaS paper + """ + + def __init__(self, reduction_type: Literal["image", "batch"] = "batch"): + super().__init__() + + self.reduction_type = reduction_type + # reduction here is different from the image/batch-based reduction. This is either "mean" or "sum" + self.mse_loss = MSELoss(reduction="none") + + def forward( + self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"] + ) -> TensorType[0]: + """ + Args: + prediction: predicted depth map + target: ground truth depth map + mask: mask of valid pixels + Returns: + mse loss based on reduction function + """ + summed_mask = torch.sum(mask, (1, 2)) + image_loss = torch.sum(self.mse_loss(prediction, target) * mask, (1, 2)) + image_loss = masked_reduction(image_loss, summed_mask, self.reduction_type) + + return image_loss + + +# losses based on https://github.com/autonomousvision/monosdf/blob/main/code/model/loss.py +class GradientLoss(nn.Module): + """ + multiscale, scale-invariant gradient matching term to the disparity space. + This term biases discontinuities to be sharp and to coincide with discontinuities in the ground truth + More info here https://arxiv.org/pdf/1907.01341.pdf Equation 11 + """ + + def __init__(self, scales: int = 4, reduction_type: Literal["image", "batch"] = "batch"): + """ + Args: + scales: number of scales to use + reduction_type: either "batch" or "image" + """ + super().__init__() + self.reduction_type = reduction_type + self.__scales = scales + + def forward( + self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"] + ) -> TensorType[0]: + """ + Args: + prediction: predicted depth map + target: ground truth depth map + mask: mask of valid pixels + Returns: + gradient loss based on reduction function + """ + total = 0 + + for scale in range(self.__scales): + step = pow(2, scale) + + grad_loss = self.gradient_loss( + prediction[:, ::step, ::step], + target[:, ::step, ::step], + mask[:, ::step, ::step], + ) + total += grad_loss + + return total + + def gradient_loss( + self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"] + ) -> TensorType[0]: + """ + multiscale, scale-invariant gradient matching term to the disparity space. + This term biases discontinuities to be sharp and to coincide with discontinuities in the ground truth + More info here https://arxiv.org/pdf/1907.01341.pdf Equation 11 + Args: + prediction: predicted depth map + target: ground truth depth map + reduction: reduction function, either reduction_batch_based or reduction_image_based + Returns: + gradient loss based on reduction function + """ + summed_mask = torch.sum(mask, (1, 2)) + diff = prediction - target + diff = torch.mul(mask, diff) + + grad_x = torch.abs(diff[:, :, 1:] - diff[:, :, :-1]) + mask_x = torch.mul(mask[:, :, 1:], mask[:, :, :-1]) + grad_x = torch.mul(mask_x, grad_x) + + grad_y = torch.abs(diff[:, 1:, :] - diff[:, :-1, :]) + mask_y = torch.mul(mask[:, 1:, :], mask[:, :-1, :]) + grad_y = torch.mul(mask_y, grad_y) + + image_loss = torch.sum(grad_x, (1, 2)) + torch.sum(grad_y, (1, 2)) + image_loss = masked_reduction(image_loss, summed_mask, self.reduction_type) + + return image_loss + + +class ScaleAndShiftInvariantLoss(nn.Module): + """ + Scale and shift invariant loss as described in + "Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer" + https://arxiv.org/pdf/1907.01341.pdf + """ + + def __init__(self, alpha: float = 0.5, scales: int = 4, reduction_type: Literal["image", "batch"] = "batch"): + """ + Args: + alpha: weight of the regularization term + scales: number of scales to use + reduction_type: either "batch" or "image" + """ + super().__init__() + self.__data_loss = MiDaSMSELoss(reduction_type=reduction_type) + self.__regularization_loss = GradientLoss(scales=scales, reduction_type=reduction_type) + self.__alpha = alpha + + self.__prediction_ssi = None + + @classmethod + def compute_scale_and_shift( + cls, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"] + ): + """ + More info here: https://arxiv.org/pdf/2206.00665.pdf supplementary section A2 Depth Consistency Loss + This function computes scale/shift required to normalizes predicted depth map, + to allow for using normalized depth maps as input from monocular depth estimation networks. + These networks are trained such that they predict normalized depth maps. + + Solves for scale/shift using a least squares approach with a closed form solution: + Based on: + https://github.com/autonomousvision/monosdf/blob/d9619e948bf3d85c6adec1a643f679e2e8e84d4b/code/model/loss.py#L7 + Args: + prediction: predicted depth map + target: ground truth depth map + mask: mask of valid pixels + Returns: + scale and shift for depth prediction + """ + # system matrix: A = [[a_00, a_01], [a_10, a_11]] + a_00 = torch.sum(mask * prediction * prediction, (1, 2)) + a_01 = torch.sum(mask * prediction, (1, 2)) + a_11 = torch.sum(mask, (1, 2)) + + # right hand side: b = [b_0, b_1] + b_0 = torch.sum(mask * prediction * target, (1, 2)) + b_1 = torch.sum(mask * target, (1, 2)) + + # solution: x = A^-1 . b = [[a_11, -a_01], [-a_10, a_00]] / (a_00 * a_11 - a_01 * a_10) . b + scale = torch.zeros_like(b_0) + shift = torch.zeros_like(b_1) + + det = a_00 * a_11 - a_01 * a_01 + valid = det.nonzero() + + scale[valid] = (a_11[valid] * b_0[valid] - a_01[valid] * b_1[valid]) / det[valid] + shift[valid] = (-a_01[valid] * b_0[valid] + a_00[valid] * b_1[valid]) / det[valid] + + return scale, shift + + def forward( + self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"] + ) -> TensorType[0]: + """ + Args: + prediction: predicted depth map (unnormalized) + target: ground truth depth map (normalized) + mask: mask of valid pixels + Returns: + scale and shift invariant loss + """ + scale, shift = ScaleAndShiftInvariantLoss.compute_scale_and_shift(prediction, target, mask) + self.__prediction_ssi = scale.view(-1, 1, 1) * prediction + shift.view(-1, 1, 1) + + total = self.__data_loss(self.__prediction_ssi, target, mask) + if self.__alpha > 0: + total += self.__alpha * self.__regularization_loss(self.__prediction_ssi, target, mask) + + return total + + def __get_prediction_ssi(self): + """ + scale and shift invariant prediction + from https://arxiv.org/pdf/1907.01341.pdf equation 1 + """ + return self.__prediction_ssi + + prediction_ssi = property(__get_prediction_ssi) diff --git a/nerfstudio/utils/math.py b/nerfstudio/utils/math.py index cee6108a9..b0a0aaec4 100644 --- a/nerfstudio/utils/math.py +++ b/nerfstudio/utils/math.py @@ -19,6 +19,7 @@ from typing import Tuple import torch from torchtyping import TensorType +from typing_extensions import Literal _USE_NERFACC = True @@ -285,3 +286,28 @@ def safe_normalize( Normalized vectors. """ return vectors / (torch.norm(vectors, dim=-1, keepdim=True) + eps) + + +def masked_reduction( + input_tensor: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"], reduction_type: Literal["image", "batch"] +): + """ + Whether to consolidate the input_tensor across the batch or across the image + Args: + input_tensor: input tensor + mask: mask tensor + reduction_type: either "batch" or "image" + Returns: + input_tensor: reduced input_tensor + """ + if reduction_type == "batch": + # avoid division by 0 (if sum(M) = sum(sum(mask)) = 0: sum(image_loss) = 0) + divisor = torch.sum(mask) + input_tensor = 0 if divisor == 0 else torch.sum(input_tensor) / divisor + elif reduction_type == "image": + # avoid division by 0 (if M = sum(mask) = 0: image_loss = 0) + valid = mask.nonzero() + + input_tensor[valid] = input_tensor[valid] / mask[valid] + input_tensor = torch.mean(input_tensor) + return input_tensor From 2357332e66db67a65966a977f940188a4d32bd77 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Wed, 15 Feb 2023 15:35:08 -0800 Subject: [PATCH 05/17] Proposal network initial sampler (#1435) Allow first proposal sampling method --- nerfstudio/model_components/ray_samplers.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/nerfstudio/model_components/ray_samplers.py b/nerfstudio/model_components/ray_samplers.py index 5df52d2fa..ef0079ffd 100644 --- a/nerfstudio/model_components/ray_samplers.py +++ b/nerfstudio/model_components/ray_samplers.py @@ -509,7 +509,16 @@ class VolumetricSampler(Sampler): class ProposalNetworkSampler(Sampler): - """Sampler that uses a proposal network to generate samples.""" + """Sampler that uses a proposal network to generate samples. + + Args: + num_proposal_samples_per_ray: Number of samples to generate per ray for each proposal step. + num_nerf_samples_per_ray: Number of samples to generate per ray for the NERF model. + num_proposal_network_iterations: Number of proposal network iterations to run. + single_jitter: Use a same random jitter for all samples along a ray. + update_sched: A function that takes the iteration number of steps between updates. + initial_sampler: Sampler to use for the first iteration. Uses UniformLinDispPiecewise if not set. + """ def __init__( self, @@ -518,6 +527,7 @@ class ProposalNetworkSampler(Sampler): num_proposal_network_iterations: int = 2, single_jitter: bool = False, update_sched: Callable = lambda x: 1, + initial_sampler: Optional[Sampler] = None, ) -> None: super().__init__() self.num_proposal_samples_per_ray = num_proposal_samples_per_ray @@ -528,7 +538,10 @@ class ProposalNetworkSampler(Sampler): raise ValueError("num_proposal_network_iterations must be >= 1") # samplers - self.initial_sampler = UniformLinDispPiecewiseSampler(single_jitter=single_jitter) + if initial_sampler is None: + self.initial_sampler = UniformLinDispPiecewiseSampler(single_jitter=single_jitter) + else: + self.initial_sampler = initial_sampler self.pdf_sampler = PDFSampler(include_original=False, single_jitter=single_jitter) self._anneal = 1.0 From 8e4ae6bebfc3a93f351d63456ddd3f3b46c515dc Mon Sep 17 00:00:00 2001 From: Sacha Lewin Date: Sat, 18 Feb 2023 05:39:10 +0100 Subject: [PATCH 06/17] Allow use of black or white background in Nerfacto models (#1441) * Allow black/white background color in Nerfacto * Use white background in Nerfacto Blender Benchmark * Fixed unused import * Updated index for license headers script --- nerfstudio/.vscode/settings.json | 3 +++ nerfstudio/model_components/renderers.py | 7 ++++--- nerfstudio/models/instant_ngp.py | 8 ++------ nerfstudio/models/nerfacto.py | 2 +- nerfstudio/models/nerfplayer_nerfacto.py | 2 +- nerfstudio/models/nerfplayer_ngp.py | 12 ++---------- scripts/benchmarking/launch_eval_blender.sh | 0 scripts/benchmarking/launch_train_blender.sh | 2 +- scripts/completions/install.py | 0 scripts/docs/add_nb_tags.py | 0 scripts/docs/build_docs.py | 0 scripts/downloads/download_data.py | 0 scripts/eval.py | 0 scripts/github/run_actions.py | 0 scripts/process_data.py | 0 scripts/train.py | 0 tests/data/lego_test/transforms_train.json | 0 17 files changed, 14 insertions(+), 22 deletions(-) create mode 100644 nerfstudio/.vscode/settings.json mode change 100755 => 100644 scripts/benchmarking/launch_eval_blender.sh mode change 100755 => 100644 scripts/benchmarking/launch_train_blender.sh mode change 100755 => 100644 scripts/completions/install.py mode change 100755 => 100644 scripts/docs/add_nb_tags.py mode change 100755 => 100644 scripts/docs/build_docs.py mode change 100755 => 100644 scripts/downloads/download_data.py mode change 100755 => 100644 scripts/eval.py mode change 100755 => 100644 scripts/github/run_actions.py mode change 100755 => 100644 scripts/process_data.py mode change 100755 => 100644 scripts/train.py mode change 100755 => 100644 tests/data/lego_test/transforms_train.json diff --git a/nerfstudio/.vscode/settings.json b/nerfstudio/.vscode/settings.json new file mode 100644 index 000000000..457f44d9b --- /dev/null +++ b/nerfstudio/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.analysis.typeCheckingMode": "basic" +} \ No newline at end of file diff --git a/nerfstudio/model_components/renderers.py b/nerfstudio/model_components/renderers.py index cf5af2a7d..3d01bddea 100644 --- a/nerfstudio/model_components/renderers.py +++ b/nerfstudio/model_components/renderers.py @@ -37,6 +37,7 @@ from torchtyping import TensorType from typing_extensions import Literal from nerfstudio.cameras.rays import RaySamples +from nerfstudio.utils import colors from nerfstudio.utils.math import components_from_spherical_harmonics, safe_normalize BACKGROUND_COLOR_OVERRIDE: Optional[TensorType[3]] = None @@ -70,7 +71,7 @@ class RGBRenderer(nn.Module): cls, rgb: TensorType["bs":..., "num_samples", 3], weights: TensorType["bs":..., "num_samples", 1], - background_color: Union[Literal["random", "black", "last_sample"], TensorType[3]] = "random", + background_color: Union[Literal["random", "white", "black", "last_sample"], TensorType[3]] = "random", ray_indices: Optional[TensorType["num_samples"]] = None, num_rays: Optional[int] = None, ) -> TensorType["bs":..., 3]: @@ -102,8 +103,8 @@ class RGBRenderer(nn.Module): background_color = rgb[..., -1, :] if background_color == "random": background_color = torch.rand_like(comp_rgb).to(rgb.device) - if background_color == "black": - background_color = torch.zeros_like(comp_rgb).to(rgb.device) + if isinstance(background_color, str) and background_color in colors.COLORS_DICT: + background_color = colors.COLORS_DICT[background_color].to(rgb.device) assert isinstance(background_color, torch.Tensor) comp_rgb = comp_rgb + background_color.to(weights.device) * (1.0 - accumulated_weight) diff --git a/nerfstudio/models/instant_ngp.py b/nerfstudio/models/instant_ngp.py index 9c0579031..ace4b77c6 100644 --- a/nerfstudio/models/instant_ngp.py +++ b/nerfstudio/models/instant_ngp.py @@ -46,7 +46,7 @@ from nerfstudio.model_components.renderers import ( RGBRenderer, ) from nerfstudio.models.base_model import Model, ModelConfig -from nerfstudio.utils import colormaps, colors +from nerfstudio.utils import colormaps @dataclass @@ -123,11 +123,7 @@ class NGPModel(Model): ) # renderers - background_color = "random" - if self.config.background_color in ["white", "black"]: - background_color = colors.COLORS_DICT[self.config.background_color] - - self.renderer_rgb = RGBRenderer(background_color=background_color) + self.renderer_rgb = RGBRenderer(background_color=self.config.background_color) self.renderer_accumulation = AccumulationRenderer() self.renderer_depth = DepthRenderer(method="expected") diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py index 4aa58d468..4972dd5be 100644 --- a/nerfstudio/models/nerfacto.py +++ b/nerfstudio/models/nerfacto.py @@ -68,7 +68,7 @@ class NerfactoModelConfig(ModelConfig): """How far along the ray to start sampling.""" far_plane: float = 1000.0 """How far along the ray to stop sampling.""" - background_color: Literal["random", "last_sample"] = "last_sample" + background_color: Literal["random", "last_sample", "black", "white"] = "last_sample" """Whether to randomize the background color.""" num_levels: int = 16 """Number of levels of the hashmap for the base mlp.""" diff --git a/nerfstudio/models/nerfplayer_nerfacto.py b/nerfstudio/models/nerfplayer_nerfacto.py index 3811aba67..e7ee3d14c 100644 --- a/nerfstudio/models/nerfplayer_nerfacto.py +++ b/nerfstudio/models/nerfplayer_nerfacto.py @@ -63,7 +63,7 @@ class NerfplayerNerfactoModelConfig(NerfactoModelConfig): """How far along the ray to start sampling.""" far_plane: float = 1000.0 """How far along the ray to stop sampling.""" - background_color: Literal["random", "last_sample"] = "random" + background_color: Literal["random", "last_sample", "black", "white"] = "random" """Whether to randomize the background color. (Random is reported to be better on DyCheck.)""" num_levels: int = 16 """Hashing grid parameter.""" diff --git a/nerfstudio/models/nerfplayer_ngp.py b/nerfstudio/models/nerfplayer_ngp.py index 878694468..09f68d67f 100644 --- a/nerfstudio/models/nerfplayer_ngp.py +++ b/nerfstudio/models/nerfplayer_ngp.py @@ -42,7 +42,6 @@ from nerfstudio.model_components.renderers import ( ) from nerfstudio.models.base_model import Model from nerfstudio.models.instant_ngp import InstantNGPModelConfig, NGPModel -from nerfstudio.utils import colors @dataclass @@ -128,13 +127,6 @@ class NerfplayerNGPModel(NGPModel): ) # need to update the density_fn later during forward (for input time) # renderers - self.train_background_color = self.config.train_background_color - self.eval_background_color = self.config.eval_background_color - if self.config.train_background_color in ["white", "black"]: - self.train_background_color = colors.COLORS_DICT[self.config.train_background_color] - if self.config.eval_background_color in ["white", "black"]: - self.eval_background_color = colors.COLORS_DICT[self.config.eval_background_color] - self.renderer_rgb = RGBRenderer() # will update bgcolor later during forward self.renderer_accumulation = AccumulationRenderer() self.renderer_depth = DepthRenderer(method="expected") @@ -175,9 +167,9 @@ class NerfplayerNGPModel(NGPModel): # update bgcolor in the renderer; usually random color for training and fixed color for inference if self.training: - self.renderer_rgb.background_color = self.train_background_color + self.renderer_rgb.background_color = self.config.train_background_color else: - self.renderer_rgb.background_color = self.eval_background_color + self.renderer_rgb.background_color = self.config.eval_background_color rgb = self.renderer_rgb( rgb=field_outputs[FieldHeadNames.RGB], weights=weights, diff --git a/scripts/benchmarking/launch_eval_blender.sh b/scripts/benchmarking/launch_eval_blender.sh old mode 100755 new mode 100644 diff --git a/scripts/benchmarking/launch_train_blender.sh b/scripts/benchmarking/launch_train_blender.sh old mode 100755 new mode 100644 index 2ddb0b008..298e21749 --- a/scripts/benchmarking/launch_train_blender.sh +++ b/scripts/benchmarking/launch_train_blender.sh @@ -28,7 +28,7 @@ fi method_opts=() if [ "$method_name" = "nerfacto" ]; then # https://github.com/nerfstudio-project/nerfstudio/issues/806#issuecomment-1284327844 - method_opts=(--pipeline.model.near-plane 2. --pipeline.model.far-plane 6. --pipeline.datamanager.camera-optimizer.mode off --pipeline.model.use-average-appearance-embedding False) + method_opts=(--pipeline.model.background-color white --pipeline.model.near-plane 2. --pipeline.model.far-plane 6. --pipeline.datamanager.camera-optimizer.mode off --pipeline.model.use-average-appearance-embedding False) fi shift $((OPTIND-1)) diff --git a/scripts/completions/install.py b/scripts/completions/install.py old mode 100755 new mode 100644 diff --git a/scripts/docs/add_nb_tags.py b/scripts/docs/add_nb_tags.py old mode 100755 new mode 100644 diff --git a/scripts/docs/build_docs.py b/scripts/docs/build_docs.py old mode 100755 new mode 100644 diff --git a/scripts/downloads/download_data.py b/scripts/downloads/download_data.py old mode 100755 new mode 100644 diff --git a/scripts/eval.py b/scripts/eval.py old mode 100755 new mode 100644 diff --git a/scripts/github/run_actions.py b/scripts/github/run_actions.py old mode 100755 new mode 100644 diff --git a/scripts/process_data.py b/scripts/process_data.py old mode 100755 new mode 100644 diff --git a/scripts/train.py b/scripts/train.py old mode 100755 new mode 100644 diff --git a/tests/data/lego_test/transforms_train.json b/tests/data/lego_test/transforms_train.json old mode 100755 new mode 100644 From 15dfc68453aeb80b62bf768000cb15d0d2925465 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Sat, 18 Feb 2023 17:13:46 -0800 Subject: [PATCH 07/17] Delete nerfstudio/.vscode directory Added by mistake. --- nerfstudio/.vscode/settings.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 nerfstudio/.vscode/settings.json diff --git a/nerfstudio/.vscode/settings.json b/nerfstudio/.vscode/settings.json deleted file mode 100644 index 457f44d9b..000000000 --- a/nerfstudio/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.analysis.typeCheckingMode": "basic" -} \ No newline at end of file From df0f3205f402c5a65b10703ba89f0bd28d8ab6df Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Sat, 18 Feb 2023 19:03:46 -0800 Subject: [PATCH 08/17] Add ns-viewer docs (#1450) --- docs/reference/cli/index.md | 2 ++ docs/reference/cli/ns_viewer.md | 9 +++++++++ 2 files changed, 11 insertions(+) create mode 100644 docs/reference/cli/ns_viewer.md diff --git a/docs/reference/cli/index.md b/docs/reference/cli/index.md index 797f0d9eb..9e1b8fae4 100644 --- a/docs/reference/cli/index.md +++ b/docs/reference/cli/index.md @@ -12,6 +12,7 @@ Here are the popular commands that we offer. If you've cloned the repo, you can | ns-process-data | Generate a dataset from your own data | scripts/process_data.py | | ns-download-data | Download existing captures | scripts/downloads/download_data.py | | ns-train | Generate a NeRF | scripts/train.py | +| ns-viewer | View a trained NeRF | scripts/viewer/run_viewer.py | | ns-eval | Run evaluation metrics for your Model | scripts/eval.py | | ns-render | Render out a video of your NeRF | scripts/render.py | | ns-export | Export a NeRF into other formats | scripts/exporter.py | @@ -24,6 +25,7 @@ ns_process_data ns_download_data ns_train ns_render +ns_viewer ns_export ns_eval ``` diff --git a/docs/reference/cli/ns_viewer.md b/docs/reference/cli/ns_viewer.md new file mode 100644 index 000000000..841472a5f --- /dev/null +++ b/docs/reference/cli/ns_viewer.md @@ -0,0 +1,9 @@ +# ns-viewer + +```{eval-rst} +.. argparse:: + :module: scripts.viewer.run_viewer + :func: get_parser_fn + :prog: ns-viewer + :nodefault: +``` From 2901e5f31a2fab50514feb877bf99d4b73bc6ad1 Mon Sep 17 00:00:00 2001 From: Nicolas Zunhammer Date: Sun, 19 Feb 2023 13:58:33 +0100 Subject: [PATCH 09/17] Bump Dockerfile versions, add user to sudoers list. (#1448) * Update Dockerfile, bump CUDA from 11.7.1 to 11.8.0, bump Colmap from 3.7 to 3.8, add user to sudoers list. * Improve description. * Fix wrong description and typos. * Add better description. * Fix build error of Dockerfile. * Bump pytorch versions. --------- Co-authored-by: Nicolas Zunhammer --- Dockerfile | 40 ++++++++++++++++++++++----------- docs/quickstart/installation.md | 14 +++++++----- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index 94587f550..a6417e419 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,16 @@ # Define base image. -FROM nvidia/cuda:11.7.1-devel-ubuntu22.04 +FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 + +# Variables used at build time. +## CUDA architectures, required by Colmap and tiny-cuda-nn. +## NOTE: All commonly used GPU architectures are included and supported here. To speedup the image build process remove all architectures but the one of your explicit GPU. Find details here: https://developer.nvidia.com/cuda-gpus (8.6 translates to 86 in the line below) or in the docs. +ARG CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37 # Set environment variables. ## Set non-interactive to prevent asking for user inputs blocking image creation. ENV DEBIAN_FRONTEND=noninteractive ## Set timezone as it is required by some packages. ENV TZ=Europe/Berlin -## CUDA architectures, required by tiny-cuda-nn. -## NOTE: All commonly used GPU architectures are included and supported here. To speedup the image build process remove all architectures but the one of your explicit GPU. Find details here: https://developer.nvidia.com/cuda-gpus (8.6 translates to 86 in the line below) or in the docs. -ENV TCNN_CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37 ## CUDA Home, required to find CUDA in some packages. ENV CUDA_HOME="/usr/local/cuda" @@ -17,6 +19,7 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends \ build-essential \ cmake \ + curl \ ffmpeg \ git \ libatlas-base-dev \ @@ -27,6 +30,7 @@ RUN apt-get update && \ libboost-test-dev \ libcgal-dev \ libeigen3-dev \ + libflann-dev \ libfreeimage-dev \ libgflags-dev \ libglew-dev \ @@ -34,15 +38,18 @@ RUN apt-get update && \ libmetis-dev \ libprotobuf-dev \ libqt5opengl5-dev \ + libsqlite3-dev \ libsuitesparse-dev \ nano \ protobuf-compiler \ python3.10-dev \ python3-pip \ qtbase5-dev \ + sudo \ wget && \ rm -rf /var/lib/apt/lists/* + # Install GLOG (required by ceres). RUN git clone --branch v0.6.0 https://github.com/google/glog.git --single-branch && \ cd glog && \ @@ -69,22 +76,28 @@ RUN git clone --branch 2.1.0 https://ceres-solver.googlesource.com/ceres-solver. rm -rf ceres-solver # Install colmap. -RUN git clone --branch 3.7 https://github.com/colmap/colmap.git --single-branch && \ +RUN git clone --branch 3.8 https://github.com/colmap/colmap.git --single-branch && \ cd colmap && \ mkdir build && \ cd build && \ cmake .. -DCUDA_ENABLED=ON \ - -DCUDA_NVCC_FLAGS="--std c++14" && \ + -DCUDA_NVCC_FLAGS="--std c++14" \ + -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES} && \ make -j && \ make install && \ cd ../.. && \ rm -rf colmap # Create non root user and setup environment. -RUN useradd -m -d /home/user -u 1000 user +RUN useradd -m -d /home/user -g root -G sudo -u 1000 user +RUN usermod -aG sudo user +# Set user password +RUN echo "user:user" | chpasswd +# Ensure sudo group users are not asked for a password when using sudo command by ammending sudoers file +RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers # Switch to new uer and workdir. -USER 1000:1000 +USER 1000 WORKDIR /home/user # Add local user binary folder to PATH variable. @@ -93,16 +106,17 @@ SHELL ["/bin/bash", "-c"] # Upgrade pip and install packages. RUN python3.10 -m pip install --upgrade pip setuptools pathtools promise -# Install pytorch and submodules. -RUN python3.10 -m pip install torch==1.12.1+cu116 torchvision==0.13.1+cu116 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu116 -# Install tynyCUDNN. +# Install pytorch and submodules (Currently, we still use cu116 which is the latest version for toch 1.12.1 and is compatible with CUDA 11.8). +RUN python3.10 -m pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116 +# Install tynyCUDNN (we need to set the target architectures as environment variable first). +ENV TCNN_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES} RUN python3.10 -m pip install git+https://github.com/NVlabs/tiny-cuda-nn.git#subdirectory=bindings/torch # Copy nerfstudio folder and give ownership to user. ADD . /home/user/nerfstudio USER root -RUN chown -R user:user /home/user/nerfstudio -USER 1000:1000 +RUN chown -R user /home/user/nerfstudio +USER 1000 # Install nerfstudio dependencies. RUN cd nerfstudio && \ diff --git a/docs/quickstart/installation.md b/docs/quickstart/installation.md index 20ab8859c..5d53a4d0c 100644 --- a/docs/quickstart/installation.md +++ b/docs/quickstart/installation.md @@ -92,8 +92,8 @@ pip install -e .[docs] ## Use docker image Instead of installing and compiling prerequisites, setting up the environment and installing dependencies, a ready to use docker image is provided. ### Prerequisites -Docker ([get docker](https://docs.docker.com/get-docker/)) and nvidia GPU drivers ([get nvidia drivers](https://www.nvidia.de/Download/index.aspx?lang=de)), capable of working with CUDA 11.7, must be installed. -The docker image can then either be pulled from [here](https://hub.docker.com/r/dromni/nerfstudio/tags) (replace with the actual version, e.g. 0.1.10) +Docker ([get docker](https://docs.docker.com/get-docker/)) and nvidia GPU drivers ([get nvidia drivers](https://www.nvidia.de/Download/index.aspx?lang=de)), capable of working with CUDA 11.8, must be installed. +The docker image can then either be pulled from [here](https://hub.docker.com/r/dromni/nerfstudio/tags) (replace with the actual version, e.g. 0.1.17) ```bash docker pull dromni/nerfstudio: ``` @@ -110,13 +110,15 @@ docker run --gpus all \ # Give the conta -p 7007:7007 \ # Map port from local machine to docker container (required to access the web interface/UI). --rm \ # Remove container after it is closed (recommended). -it \ # Start container in interactive mode. - nerfstudio # Docker image name + dromni/nerfstudio: # Docker image name if you pulled from docker hub. + <--- OR ---> + nerfstudio # Docker image tag if you built the image from the Dockerfile by yourself using the command from above. ``` ### Call nerfstudio commands directly Besides, the container can also directly be used by adding the nerfstudio command to the end. ```bash docker run --gpus all -v /folder/of/your/data:/workspace/ -v /home//.cache/:/home/user/.cache/ -p 7007:7007 --rm -it # Parameters. - nerfstudio \ # Docker image name + dromni/nerfstudio: \ # Docker image name ns-process-data video --data /workspace/video.mp4 # Smaple command of nerfstudio. ``` ### Note @@ -125,10 +127,10 @@ docker run --gpus all -v /folder/of/your/data:/workspace/ -v /home//. - Always use full paths, relative paths are known to create issues when being used in mounts into docker. - Everything inside the container, what is not in a mounted folder (workspace in the above example), will be permanently removed after destroying the container. Always do all your tasks and output folder in workdir! - The user inside the container is called 'user' and is mapped to the local user with ID 1000 (usually the first non-root user on Linux systems). -- The container currently is based on nvidia/cuda:11.7.1-devel-ubuntu22.04, consequently it comes with CUDA 11.7 which must be supported by the nvidia driver. No local CUDA installation is required or will be affected by using the docker image. +- The container currently is based on nvidia/cuda:11.8.0-devel-ubuntu22.04, consequently it comes with CUDA 11.8 which must be supported by the nvidia driver. No local CUDA installation is required or will be affected by using the docker image. - The docker image (respectively Ubuntu 22.04) comes with Python3.10, no older version of Python is installed. - If you call the container with commands directly, you still might want to add the interactive terminal ('-it') flag to get live log outputs of the nerfstudio scripts. In case the container is used in an automated environment the flag should be discarded. -- The current version of docker is built for multi-architecture (CUDA architectures) use. The target architecture must be defined at build time for tinyCUDNN to be able to compile properly. If your GPU architecture is not covered by the following table you need to replace the number in the line ```ENV TCNN_CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37``` to you specific architecture. It also is a good idea to remove all architectures but yours (e.g. ```ENV TCNN_CUDA_ARCHITECTURES=86```) to speedup the docker build a lot. +- The current version of docker is built for multi-architecture (CUDA architectures) use. The target architecture(s) must be defined at build time for Colmap and tinyCUDNN to be able to compile properly. If your GPU architecture is not covered by the following table you need to replace the number in the line ```ARG CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37``` to your specific architecture. It also is a good idea to remove all architectures but yours (e.g. ```ARG CUDA_ARCHITECTURES=86```) to speedup the docker build process a lot. **Currently supported CUDA architectures in the docker image** From 73f0f5a16a107dccc7b2d64f3bb686a456d1f3d5 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Sun, 19 Feb 2023 13:43:43 -0800 Subject: [PATCH 10/17] Fix mesh export normals (#1452) * Fix mesh export normals * lint --- nerfstudio/exporter/exporter_utils.py | 4 ++++ nerfstudio/models/nerfplayer_nerfacto.py | 12 ++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/nerfstudio/exporter/exporter_utils.py b/nerfstudio/exporter/exporter_utils.py index eb6c5f9b6..9f4168517 100644 --- a/nerfstudio/exporter/exporter_utils.py +++ b/nerfstudio/exporter/exporter_utils.py @@ -151,6 +151,10 @@ def generate_point_cloud( CONSOLE.print(f"Please set --normal_output_name to one of: {outputs.keys()}", justify="center") sys.exit(1) normal = outputs[normal_output_name] + assert ( + torch.min(normal) >= 0.0 and torch.max(normal) <= 1.0 + ), "Normal values from method output must be in [0, 1]" + normal = (normal * 2.0) - 1.0 point = ray_bundle.origins + ray_bundle.directions * depth if use_bounding_box: diff --git a/nerfstudio/models/nerfplayer_nerfacto.py b/nerfstudio/models/nerfplayer_nerfacto.py index e7ee3d14c..e10b3cad2 100644 --- a/nerfstudio/models/nerfplayer_nerfacto.py +++ b/nerfstudio/models/nerfplayer_nerfacto.py @@ -50,6 +50,7 @@ from nerfstudio.model_components.renderers import ( RGBRenderer, ) from nerfstudio.model_components.scene_colliders import NearFarCollider +from nerfstudio.model_components.shaders import NormalsShader from nerfstudio.models.base_model import Model from nerfstudio.models.nerfacto import NerfactoModel, NerfactoModelConfig @@ -162,6 +163,9 @@ class NerfplayerNerfactoModel(NerfactoModel): self.renderer_depth = DepthRenderer(method="expected") # for depth loss self.renderer_normals = NormalsRenderer() + # shaders + self.normals_shader = NormalsShader() + # losses self.rgb_loss = MSELoss() @@ -192,8 +196,12 @@ class NerfplayerNerfactoModel(NerfactoModel): } if self.config.predict_normals: - outputs["normals"] = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights) - outputs["pred_normals"] = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights) + outputs["normals"] = self.normals_shader( + self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights) + ) + outputs["pred_normals"] = self.normals_shader( + self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights) + ) # These use a lot of GPU memory, so we avoid storing them for eval. if self.training: From ceda935e64d4393ec0024a3db0fe35dab18fe638 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Sun, 19 Feb 2023 13:53:01 -0800 Subject: [PATCH 11/17] Fix depth nerfacto exporting (#1453) --- nerfstudio/models/depth_nerfacto.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nerfstudio/models/depth_nerfacto.py b/nerfstudio/models/depth_nerfacto.py index 2bea84c7b..683c89e1f 100644 --- a/nerfstudio/models/depth_nerfacto.py +++ b/nerfstudio/models/depth_nerfacto.py @@ -70,7 +70,8 @@ class DepthNerfactoModel(NerfactoModel): def get_outputs(self, ray_bundle: RayBundle): outputs = super().get_outputs(ray_bundle) - outputs["directions_norm"] = ray_bundle.metadata["directions_norm"] + if ray_bundle.metadata is not None and "directions_norm" in ray_bundle.metadata: + outputs["directions_norm"] = ray_bundle.metadata["directions_norm"] return outputs def get_metrics_dict(self, outputs, batch): From e89d113ddceccf8fef6d5a5839108b91258cd442 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Sun, 19 Feb 2023 14:48:31 -0800 Subject: [PATCH 12/17] Fix colmap bug when not all images are matched (#1454) Fix colmap when not 100 matching --- nerfstudio/process_data/colmap_utils.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/nerfstudio/process_data/colmap_utils.py b/nerfstudio/process_data/colmap_utils.py index 93f47adfa..9db20647f 100644 --- a/nerfstudio/process_data/colmap_utils.py +++ b/nerfstudio/process_data/colmap_utils.py @@ -604,14 +604,6 @@ def colmap_to_json( cameras = read_cameras_binary(cameras_path) images = read_images_binary(images_path) - # Images were renamed to frame_{i:05d}.{ext} and - # the filenames needs to be replaced in the transforms.json as well - original_filenames = [x.name for x in images.values()] - # Sort was used in nerfstudio.process_data.process_data_utils:get_image_filenames - original_filenames.sort() - # Build the map to the new filenames - filename_map = {name: f"frame_{i+1:05d}{os.path.splitext(name)[-1]}" for i, name in enumerate(original_filenames)} - # Only supports one camera camera_params = cameras[1].params @@ -627,8 +619,7 @@ def colmap_to_json( c2w = c2w[np.array([1, 0, 2, 3]), :] c2w[2, :] *= -1 - name = filename_map[im_data.name] - name = Path(f"./images/{name}") + name = Path(f"./images/{im_data.name}") frame = { "file_path": name.as_posix(), From e2dadc20ab5be3ba41ed1fe104c37272f3631018 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Sun, 19 Feb 2023 14:49:59 -0800 Subject: [PATCH 13/17] v0.1.18 (#1455) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8901aa282..73bdc21a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "nerfstudio" -version = "0.1.17" +version = "0.1.18" description = "All-in-one repository for state-of-the-art NeRFs" readme = "README.md" license = { text="Apache 2.0"} From f8e8d8266ed7b34fb98227cd214f2e93109c036e Mon Sep 17 00:00:00 2001 From: Alexander Kristoffersen Date: Sun, 19 Feb 2023 22:14:27 -0800 Subject: [PATCH 14/17] Patch Sampling (#1458) * patch sampling. * changed str to literal. * clean up sample selector. --- .../data/datamanagers/base_datamanager.py | 11 +++- nerfstudio/data/pixel_samplers.py | 51 +++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index 1c8c3e08d..e48218c39 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -51,7 +51,11 @@ from nerfstudio.data.dataparsers.phototourism_dataparser import ( ) from nerfstudio.data.dataparsers.sdfstudio_dataparser import SDFStudioDataParserConfig from nerfstudio.data.datasets.base_dataset import InputDataset -from nerfstudio.data.pixel_samplers import EquirectangularPixelSampler, PixelSampler +from nerfstudio.data.pixel_samplers import ( + EquirectangularPixelSampler, + PatchPixelSampler, + PixelSampler, +) from nerfstudio.data.utils.dataloaders import ( CacheDataloader, FixedIndicesEvalDataloader, @@ -283,6 +287,8 @@ class VanillaDataManagerConfig(InstantiateConfig): """The scale factor for scaling spatial data such as images, mask, semantics along with relevant information about camera intrinsics """ + patch_size: int = 1 + """Size of patch to sample from. If >1, patch-based sampling will be used.""" class VanillaDataManager(DataManager): # pylint: disable=abstract-method @@ -346,6 +352,9 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method self, dataset: InputDataset, *args: Any, **kwargs: Any ) -> PixelSampler: """Infer pixel sampler to use.""" + if self.config.patch_size > 1: + return PatchPixelSampler(*args, **kwargs, patch_size=self.config.patch_size) + # If all images are equirectangular, use equirectangular pixel sampler is_equirectangular = dataset.cameras.camera_type == CameraType.EQUIRECTANGULAR.value if is_equirectangular.all(): diff --git a/nerfstudio/data/pixel_samplers.py b/nerfstudio/data/pixel_samplers.py index 91fc6301d..8ac8e183d 100644 --- a/nerfstudio/data/pixel_samplers.py +++ b/nerfstudio/data/pixel_samplers.py @@ -243,3 +243,54 @@ class EquirectangularPixelSampler(PixelSampler): # pylint: disable=too-few-publ ).long() return indices + + +class PatchPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods + """Samples 'pixel_batch's from 'image_batch's. Samples square patches + from the images randomly. Useful for patch-based losses. + + Args: + num_rays_per_batch: number of rays to sample per batch + keep_full_image: whether or not to include a reference to the full image in returned batch + patch_size: side length of patch. This must be consistent in the method + config in order for samples to be reshaped into patches correctly. + """ + + def __init__(self, num_rays_per_batch: int, keep_full_image: bool = False, **kwargs) -> None: + self.patch_size = kwargs["patch_size"] + num_rays = (num_rays_per_batch // (self.patch_size**2)) * (self.patch_size**2) + super().__init__(num_rays, keep_full_image, **kwargs) + + # overrides base method + def sample_method( # pylint: disable=no-self-use + self, + batch_size: int, + num_images: int, + image_height: int, + image_width: int, + mask: Optional[TensorType] = None, + device: Union[torch.device, str] = "cpu", + ) -> TensorType["batch_size", 3]: + + if mask: + # Note: if there is a mask, sampling reduces back to uniform sampling + indices = super().sample_method(batch_size, num_images, image_height, image_width, mask=mask, device=device) + else: + sub_bs = batch_size // (self.patch_size**2) + indices = torch.rand((sub_bs, 3), device=device) * torch.tensor( + [num_images, image_height - self.patch_size, image_width - self.patch_size], + device=device, + ) + + indices = indices.view(sub_bs, 1, 1, 3).broadcast_to(sub_bs, self.patch_size, self.patch_size, 3).clone() + + yys, xxs = torch.meshgrid( + torch.arange(self.patch_size, device=device), torch.arange(self.patch_size, device=device) + ) + indices[:, ..., 1] += yys + indices[:, ..., 2] += xxs + + indices = torch.floor(indices).long() + indices = indices.flatten(0, 2) + + return indices From 8754beeb852dd1996974cc8c77a6a5892f5bcd0f Mon Sep 17 00:00:00 2001 From: Matt Tancik Date: Mon, 20 Feb 2023 10:05:09 -0800 Subject: [PATCH 15/17] cleanup --- nerfstudio/engine/trainer.py | 6 +- nerfstudio/generative/stable_diffusion.py | 13 ++- .../generative/stable_diffusion_utils.py | 3 +- nerfstudio/models/dreamfusion.py | 28 ++--- nerfstudio/pipelines/base_pipeline.py | 2 +- nerfstudio/viewer/server/viewer_utils.py | 2 +- pyproject.toml | 2 +- scripts/generative/trace_stable_diffusion.py | 105 +++++++++++------- scripts/trace_stablediff.py | 82 -------------- 9 files changed, 97 insertions(+), 146 deletions(-) delete mode 100644 scripts/trace_stablediff.py diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index b59ce5664..91722bf25 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -146,7 +146,11 @@ class Trainer: 'inference': does not load any dataset into memory """ self.pipeline = self.config.pipeline.setup( - device=self.device, test_mode=test_mode, world_size=self.world_size, local_rank=self.local_rank, grad_scaler=self.grad_scaler + device=self.device, + test_mode=test_mode, + world_size=self.world_size, + local_rank=self.local_rank, + grad_scaler=self.grad_scaler, ) self.optimizers = self.setup_optimizers() diff --git a/nerfstudio/generative/stable_diffusion.py b/nerfstudio/generative/stable_diffusion.py index 0af0f23f6..a3955712b 100644 --- a/nerfstudio/generative/stable_diffusion.py +++ b/nerfstudio/generative/stable_diffusion.py @@ -49,11 +49,12 @@ logging.set_verbosity_error() IMG_DIM = 512 CONST_SCALE = 0.18215 SD_IDENTIFIERS = { - '1-5':'runwayml/stable-diffusion-v1-5', - '2-0':'stabilityai/stable-diffusion-2-base', - '2-1':'stabilityai/stable-diffusion-2-1-base' + "1-5": "runwayml/stable-diffusion-v1-5", + "2-0": "stabilityai/stable-diffusion-2-base", + "2-1": "stabilityai/stable-diffusion-2-1-base", } + @dataclass class UNet2DConditionOutput: """Class to hold traced model""" @@ -88,7 +89,7 @@ class StableDiffusion(nn.Module): num_train_timesteps: number of training timesteps """ - def __init__(self, device: Union[torch.device, str], num_train_timesteps: int = 1000, version='1-5') -> None: + def __init__(self, device: Union[torch.device, str], num_train_timesteps: int = 1000, version="1-5") -> None: super().__init__() self.device = device @@ -113,8 +114,8 @@ class StableDiffusion(nn.Module): pipe.enable_attention_slicing() # use jitted unet - filename_sd_id = sd_id.split('/')[-1] - unet_traced_filename = Path(appdirs.user_data_dir("nerfstudio")) / f'{filename_sd_id}_unet_traced.pt' + filename_sd_id = sd_id.split("/")[-1] + unet_traced_filename = Path(appdirs.user_data_dir("nerfstudio")) / f"{filename_sd_id}_unet_traced.pt" if unet_traced_filename.exists(): CONSOLE.print("Loading traced UNet.") unet_traced = torch.jit.load(unet_traced_filename) diff --git a/nerfstudio/generative/stable_diffusion_utils.py b/nerfstudio/generative/stable_diffusion_utils.py index 82f61ff7a..187b1a60a 100644 --- a/nerfstudio/generative/stable_diffusion_utils.py +++ b/nerfstudio/generative/stable_diffusion_utils.py @@ -16,6 +16,7 @@ Stable diffusion utils """ + def get_text_embedding(batch, prompting_type, text_embeddings, sd_device): """""" @@ -51,7 +52,7 @@ def get_text_embedding(batch, prompting_type, text_embeddings, sd_device): text_embedding = (horiz - 180) * side + (270 - horiz) * back else: # batch["central"] > 270 and batch["central"] <= 360: text_embedding = (horiz - 270) * front + (360 - horiz) * side - + text_embedding = text_embedding / 90.0 text_embedding = (vert * text_embedding + (90 - vert) * top) / 90.0 diff --git a/nerfstudio/models/dreamfusion.py b/nerfstudio/models/dreamfusion.py index 6eba9dc96..d3a658851 100644 --- a/nerfstudio/models/dreamfusion.py +++ b/nerfstudio/models/dreamfusion.py @@ -19,12 +19,12 @@ DreamFusion implementation. from __future__ import annotations from dataclasses import dataclass, field -from typing import Dict, List, Tuple, Type, Optional +from typing import Dict, List, Optional, Tuple, Type import numpy as np import torch -from torch.nn import Parameter from torch.cuda.amp.grad_scaler import GradScaler +from torch.nn import Parameter from nerfstudio.cameras.rays import RayBundle from nerfstudio.engine.callbacks import ( @@ -32,11 +32,11 @@ from nerfstudio.engine.callbacks import ( TrainingCallbackAttributes, TrainingCallbackLocation, ) -from nerfstudio.generative.stable_diffusion import StableDiffusion -from nerfstudio.generative.stable_diffusion_utils import get_text_embedding from nerfstudio.field_components.field_heads import FieldHeadNames from nerfstudio.fields.density_fields import HashMLPDensityField from nerfstudio.fields.dreamfusion_field import DreamFusionField +from nerfstudio.generative.stable_diffusion import StableDiffusion +from nerfstudio.generative.stable_diffusion_utils import get_text_embedding from nerfstudio.model_components.losses import ( MSELoss, distortion_loss, @@ -149,7 +149,8 @@ class DreamFusionModelConfig(ModelConfig): """guidance scale for sds loss""" stablediffusion_device: Optional[str] = None """device for stable diffusion""" - sd_version: str = '1-5' + sd_version: str = "1-5" + class DreamFusionModel(Model): """DreamFusionModel Model @@ -174,17 +175,17 @@ class DreamFusionModel(Model): self.random_background = config.random_background self.density_strength = 1.0 self.target_transmittance = config.target_transmittance_start - self.grad_scaler = kwargs['grad_scaler'] + self.grad_scaler = kwargs["grad_scaler"] self.prompting_type = config.prompting_type self.guidance_scale = config.guidance_scale self.top_prompt = config.top_prompt self.side_prompt = config.side_prompt - self.back_prompt = config.back_prompt + self.back_prompt = config.back_prompt self.front_prompt = config.front_prompt self.sd_device = ( - torch.device(kwargs['device']) + torch.device(kwargs["device"]) if config.stablediffusion_device is None else torch.device(config.stablediffusion_device) ) @@ -201,7 +202,7 @@ class DreamFusionModel(Model): "front_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.front_prompt}", ""), "side_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.side_prompt}", ""), "back_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.back_prompt}", ""), - "base_text_embedding": self.sd.get_text_embeds(self.cur_prompt, "") + "base_text_embedding": self.sd.get_text_embeds(self.cur_prompt, ""), } # setting up fields @@ -468,13 +469,14 @@ class DreamFusionModel(Model): "front_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.front_prompt}", ""), "side_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.side_prompt}", ""), "back_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.back_prompt}", ""), - "base_text_embedding": self.sd.get_text_embeds(self.cur_prompt, "") + "base_text_embedding": self.sd.get_text_embeds(self.cur_prompt, ""), } - + text_embedding = get_text_embedding(batch, self.prompting_type, self.text_embeddings, self.sd_device) - + train_output = ( - outputs["train_output"].view(1, int(outputs["train_output"].shape[0] ** 0.5), int(outputs["train_output"].shape[0] ** 0.5), 3) + outputs["train_output"] + .view(1, int(outputs["train_output"].shape[0] ** 0.5), int(outputs["train_output"].shape[0] ** 0.5), 3) .permute(0, 3, 1, 2) ) diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py index ac1b73dd6..bf4d400f8 100644 --- a/nerfstudio/pipelines/base_pipeline.py +++ b/nerfstudio/pipelines/base_pipeline.py @@ -237,7 +237,7 @@ class VanillaPipeline(Pipeline): num_train_data=len(self.datamanager.train_dataset), metadata=self.datamanager.train_dataset.metadata, device=device, - grad_scaler=grad_scaler + grad_scaler=grad_scaler, ) self.model.to(device) diff --git a/nerfstudio/viewer/server/viewer_utils.py b/nerfstudio/viewer/server/viewer_utils.py index 2613934d6..bbdd1099b 100644 --- a/nerfstudio/viewer/server/viewer_utils.py +++ b/nerfstudio/viewer/server/viewer_utils.py @@ -494,7 +494,7 @@ class ViewerState: prompt = getattr(graph, "prompt", None) is_generative = prompt is not None - + if is_generative: self.vis["model/is_generative"].write(str(is_generative).lower()) diff --git a/pyproject.toml b/pyproject.toml index 99c6f8ca2..f1154fcd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,7 +100,7 @@ ns-install-cli = "scripts.completions.install:entrypoint" ns-process-data = "scripts.process_data:entrypoint" ns-download-data = "scripts.downloads.download_data:entrypoint" ns-train = "scripts.train:entrypoint" -ns-trace-sd = "scripts.trace_stablediff:entrypoint" +ns-trace-sd = "scripts.generative.trace_stable_diffusion:entrypoint" ns-viewer = "scripts.viewer.run_viewer:entrypoint" ns-eval = "scripts.eval:entrypoint" ns-render = "scripts.render:entrypoint" diff --git a/scripts/generative/trace_stable_diffusion.py b/scripts/generative/trace_stable_diffusion.py index a501bd424..eee069eb3 100644 --- a/scripts/generative/trace_stable_diffusion.py +++ b/scripts/generative/trace_stable_diffusion.py @@ -2,6 +2,7 @@ import functools import sys +from dataclasses import dataclass from pathlib import Path import appdirs @@ -9,6 +10,8 @@ import torch import tyro from rich.console import Console +from nerfstudio.generative.stable_diffusion import SD_IDENTIFIERS + CONSOLE = Console(width=120) try: @@ -20,55 +23,77 @@ except ImportError: sys.exit(1) -def jit_unet(save_dir: Path = Path(appdirs.user_data_dir("nerfstudio"))): - """Trace Stable Diffusion UNet for speed improvement +@dataclass +class TraceSD: + """Traces the Stable Diffusion U-Net for better performance.""" - Args: - save_dir: directory to save the traced model - """ + sd_version: str + """Stable diffusion version.""" + save_dir: Path = Path(appdirs.user_data_dir("nerfstudio")) + """Directory to save the traced model.""" - # torch disable grad - torch.set_grad_enabled(False) + def main(self): + """Trace the Stable Diffusion U-Net and save it to disk.""" + # torch disable grad + torch.set_grad_enabled(False) - # load inputs - def generate_inputs(): - sample = torch.randn(2, 4, 64, 64).half().cuda() - timestep = torch.rand(1).half().cuda() * 999 - encoder_hidden_states = torch.randn(2, 77, 768).half().cuda() - return sample, timestep, encoder_hidden_states + if self.sd_version not in SD_IDENTIFIERS: + CONSOLE.print(f"[bold red]Invalid Stable Diffusion version; choose from {set(SD_IDENTIFIERS.keys())}") + sys.exit(1) - pipe = StableDiffusionPipeline.from_pretrained( - "runwayml/stable-diffusion-v1-5", - torch_dtype=torch.float16, - ).to("cuda") + sd_id = SD_IDENTIFIERS[self.sd_version] + pipe = StableDiffusionPipeline.from_pretrained( + sd_id, + torch_dtype=torch.float16, + ) + assert pipe is not None + pipe = pipe.to("cuda") + pipe.enable_attention_slicing() - pipe.enable_attention_slicing() + # load inputs + def generate_inputs(): + sample = torch.randn(2, 4, 64, 64).half().cuda() + timestep = torch.rand(1).half().cuda() * 999 - unet = pipe.unet - unet.eval() - unet.to(memory_format=torch.channels_last) # use channels_last memory format - unet.forward = functools.partial(unet.forward, return_dict=False) # set return_dict=False as default + # get shape of encoder hidden state (varies by SD version) + text_input = pipe.tokenizer("", return_tensors="pt") + _, w, d = pipe.text_encoder(text_input.input_ids.cuda())[0].shape - # warmup - inputs = None - for _ in range(3): - with torch.inference_mode(): - inputs = generate_inputs() - _ = unet(*inputs) + encoder_hidden_states = torch.randn(2, w, d).half().cuda() + return sample, timestep, encoder_hidden_states - # trace - print("tracing..") - assert inputs is not None - unet_traced = torch.jit.trace(unet, inputs) - unet_traced.eval() - print("done tracing") + unet = pipe.unet + unet.eval() + unet.to(memory_format=torch.channels_last) # use channels_last memory format + unet.forward = functools.partial(unet.forward, return_dict=False) # set return_dict=False as default - # save the traced model - unet_traced_filename = save_dir / "sd_unet_traced.pt" - if not unet_traced_filename.exists(): - unet_traced_filename.parent.mkdir(parents=True, exist_ok=True) - unet_traced.save(unet_traced_filename) + # warmup + inputs = None + for _ in range(3): + with torch.inference_mode(): + inputs = generate_inputs() + _ = unet(*inputs) + + # trace + print("tracing..") + assert inputs is not None + unet_traced = torch.jit.trace(unet, inputs) + unet_traced.eval() # type: ignore + print("done tracing") + + # save the traced model + filename_sd_id = sd_id.split("/")[-1] + unet_traced_filename = self.save_dir / f"{filename_sd_id}_unet_traced.pt" + if not unet_traced_filename.exists(): + unet_traced_filename.parent.mkdir(parents=True, exist_ok=True) + unet_traced.save(unet_traced_filename) # type: ignore + + +def entrypoint(): + """Entrypoint for use with pyproject scripts.""" + tyro.extras.set_accent_color("bright_yellow") + tyro.cli(TraceSD).main() if __name__ == "__main__": - tyro.cli(jit_unet) + entrypoint() diff --git a/scripts/trace_stablediff.py b/scripts/trace_stablediff.py deleted file mode 100644 index 44193307d..000000000 --- a/scripts/trace_stablediff.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Trace Stable Diffusion UNet for speed improvement""" - -import functools -import sys -from dataclasses import dataclass -from pathlib import Path - -import appdirs -import torch -import tyro -from diffusers import StableDiffusionPipeline -from rich.console import Console - -from nerfstudio.generative.stable_diffusion import SD_IDENTIFIERS - -CONSOLE = Console(width=120) - -@dataclass -class TraceSD: - """Traces the Stable Diffusion U-Net for better performance.""" - - sd_version: str - """Stable diffusion version.""" - - def main(self): - # torch disable grad - torch.set_grad_enabled(False) - - if self.sd_version not in SD_IDENTIFIERS.keys(): - CONSOLE.print(f"[bold red]Invalid Stable Diffusion version; choose from {set(SD_IDENTIFIERS.keys())}") - sys.exit(1) - - sd_id = SD_IDENTIFIERS[self.sd_version] - pipe = StableDiffusionPipeline.from_pretrained( - sd_id, - torch_dtype=torch.float16, - ).to("cuda") - pipe.enable_attention_slicing() - - # load inputs - def generate_inputs(): - sample = torch.randn(2, 4, 64, 64).half().cuda() - timestep = torch.rand(1).half().cuda() * 999 - - # get shape of encoder hidden state (varies by SD version) - text_input = pipe.tokenizer("", return_tensors="pt") - _, w, d = pipe.text_encoder(text_input.input_ids.cuda())[0].shape - - encoder_hidden_states = torch.randn(2, w, d).half().cuda() - return sample, timestep, encoder_hidden_states - - unet = pipe.unet - unet.eval() - unet.to(memory_format=torch.channels_last) # use channels_last memory format - unet.forward = functools.partial(unet.forward, return_dict=False) # set return_dict=False as default - - # warmup - for _ in range(3): - with torch.inference_mode(): - inputs = generate_inputs() - _ = unet(*inputs) - - # trace - print("tracing..") - unet_traced = torch.jit.trace(unet, inputs) - unet_traced.eval() - print("done tracing") - - # save the traced model - filename_sd_id = sd_id.split('/')[-1] - unet_traced_filename = Path(appdirs.user_data_dir("nerfstudio")) / f'{filename_sd_id}_unet_traced.pt' - if not unet_traced_filename.exists(): - unet_traced_filename.parent.mkdir(parents=True, exist_ok=True) - unet_traced.save(unet_traced_filename) - -def entrypoint(): - """Entrypoint for use with pyproject scripts.""" - tyro.extras.set_accent_color("bright_yellow") - tyro.cli(TraceSD).main() - -if __name__ == "__main__": - entrypoint() From 410b981205295e41143c539c229fa6f3bd408ec2 Mon Sep 17 00:00:00 2001 From: Matt Tancik Date: Mon, 20 Feb 2023 18:12:18 -0800 Subject: [PATCH 16/17] cleanup --- nerfstudio/configs/method_configs.py | 2 +- nerfstudio/fields/dreamfusion_field.py | 1 - nerfstudio/generative/stable_diffusion.py | 44 +++---- .../generative/stable_diffusion_utils.py | 118 ++++++++++++------ nerfstudio/models/dreamfusion.py | 74 +++++------ nerfstudio/viewer/server/viewer_utils.py | 2 + 6 files changed, 148 insertions(+), 93 deletions(-) diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index dc9aba07f..bba2f3df4 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -365,7 +365,7 @@ method_configs["dreamfusion"] = TrainerConfig( start_lambertian_training=1000, start_normals_training=500, opacity_loss_mult=0.001, - prompting_type="location_based", + positional_prompting="discrete", guidance_scale=100, ), ), diff --git a/nerfstudio/fields/dreamfusion_field.py b/nerfstudio/fields/dreamfusion_field.py index 4960fbe48..9c78513fc 100644 --- a/nerfstudio/fields/dreamfusion_field.py +++ b/nerfstudio/fields/dreamfusion_field.py @@ -29,7 +29,6 @@ from nerfstudio.data.scene_box import SceneBox from nerfstudio.field_components.activations import trunc_exp from nerfstudio.field_components.field_heads import FieldHeadNames, PredNormalsFieldHead from nerfstudio.fields.base_field import Field -from nerfstudio.utils import math try: import tinycudann as tcnn diff --git a/nerfstudio/generative/stable_diffusion.py b/nerfstudio/generative/stable_diffusion.py index a3955712b..51e03cecb 100644 --- a/nerfstudio/generative/stable_diffusion.py +++ b/nerfstudio/generative/stable_diffusion.py @@ -199,32 +199,34 @@ class StableDiffusion(nn.Module): Returns: The loss """ - image = F.interpolate(image, (IMG_DIM, IMG_DIM), mode="bilinear") - t = torch.randint(self.min_step, self.max_step + 1, [1], dtype=torch.long, device=self.device) - latents = self.imgs_to_latent(image) + # Disable autocast when using diffusers pipeline. + with torch.autocast(device_type="cuda", enabled=False): + image = F.interpolate(image.half(), (IMG_DIM, IMG_DIM), mode="bilinear") + t = torch.randint(self.min_step, self.max_step + 1, [1], dtype=torch.long, device=self.device) + latents = self.imgs_to_latent(image) - # predict the noise residual with unet, NO grad! - with torch.no_grad(): - # add noise - noise = torch.randn_like(latents) - latents_noisy = self.scheduler.add_noise(latents, noise, t) # type: ignore - # pred noise - latent_model_input = torch.cat([latents_noisy] * 2) - noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings).sample + # predict the noise residual with unet, NO grad! + with torch.no_grad(): + # add noise + noise = torch.randn_like(latents) + latents_noisy = self.scheduler.add_noise(latents, noise, t) # type: ignore + # pred noise + latent_model_input = torch.cat([latents_noisy] * 2) + noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings).sample - # perform guidance - noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) - noise_pred = noise_pred_text + guidance_scale * (noise_pred_text - noise_pred_uncond) + # perform guidance + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_text + guidance_scale * (noise_pred_text - noise_pred_uncond) - # w(t), sigma_t^2 - w = 1 - self.alphas[t] + # w(t), sigma_t^2 + w = 1 - self.alphas[t] - grad = w * (noise_pred - noise) - grad = torch.nan_to_num(grad) + grad = w * (noise_pred - noise) + grad = torch.nan_to_num(grad) - if grad_scaler is not None: - latents = grad_scaler.scale(latents) - loss = _SDSGradient.apply(latents, grad) + if grad_scaler is not None: + latents = grad_scaler.scale(latents) + loss = _SDSGradient.apply(latents, grad) return loss diff --git a/nerfstudio/generative/stable_diffusion_utils.py b/nerfstudio/generative/stable_diffusion_utils.py index 187b1a60a..d7a94805e 100644 --- a/nerfstudio/generative/stable_diffusion_utils.py +++ b/nerfstudio/generative/stable_diffusion_utils.py @@ -16,47 +16,95 @@ Stable diffusion utils """ +from torchtyping import TensorType +from typing_extensions import Literal -def get_text_embedding(batch, prompting_type, text_embeddings, sd_device): - """""" +from nerfstudio.generative.stable_diffusion import StableDiffusion - base = text_embeddings["base_text_embedding"] - front = text_embeddings["front_text_embedding"] - side = text_embeddings["side_text_embedding"] - top = text_embeddings["top_text_embedding"] - back = text_embeddings["back_text_embedding"] - assert prompting_type in ("location_based", "interpolated", "base") +class PositionalTextEmbeddings: + """Postional Prompts. Currently the following location based text embeddings are supported: + - discrete: Choose the embedding closest to the camera position + - interpolated: Interpolate between the embeddings based on the camera position + - off: Don't modify the text embedding based on the camera position - if prompting_type == "location_based": - if batch["vertical"] < 40: - text_embedding = top - elif batch["central"] > 315 or batch["central"] <= 45: - text_embedding = front - elif batch["central"] > 45 and batch["central"] <= 135: - text_embedding = side - elif batch["central"] > 135 and batch["central"] <= 225: - text_embedding = back - else: # batch["central"] > 225 and batch["central"] <= 315: - text_embedding = side + Args: + base_prompt: Prompt for base view + top_prompt: Prompt for top view + side_prompt: Prompt for side view + back_prompt: Prompt for back view + front_prompt: Prompt for front view + stable_diffusion: Instance of StableDiffusion + positional_prompting: how to incorporate position into prompt. + """ - elif prompting_type == "interpolated": - horiz = batch["central"].to(sd_device) - vert = max(batch["vertical"].to(sd_device), 0) + def __init__( + self, + base_prompt: str, + top_prompt: str, + side_prompt: str, + back_prompt: str, + front_prompt: str, + stable_diffusion: StableDiffusion, + positional_prompting: Literal["discrete", "interpolated", "off"] = "discrete", + ): + self.positional_prompting = positional_prompting + self.sd_device = stable_diffusion.device + self.sd = stable_diffusion + self.update_prompt(base_prompt, top_prompt, side_prompt, back_prompt, front_prompt) - if batch["central"] > 0 or batch["central"] <= 90: - text_embedding = (horiz) * side + (90 - horiz) * front - elif batch["central"] > 90 and batch["central"] <= 180: - text_embedding = (horiz - 90) * back + (180 - horiz) * side - elif batch["central"] > 180 and batch["central"] <= 270: - text_embedding = (horiz - 180) * side + (270 - horiz) * back - else: # batch["central"] > 270 and batch["central"] <= 360: - text_embedding = (horiz - 270) * front + (360 - horiz) * side + def update_prompt(self, base_prompt: str, top_prompt: str, side_prompt: str, back_prompt: str, front_prompt: str): + """Update the text embeddings based on the new prompts. - text_embedding = text_embedding / 90.0 - text_embedding = (vert * text_embedding + (90 - vert) * top) / 90.0 + Args: + base_prompt: Prompt for base view + top_prompt: Prompt for top view + side_prompt: Prompt for side view + back_prompt: Prompt for back view + front_prompt: Prompt for front view + """ + self.base_embed = self.sd.get_text_embeds(base_prompt, "") + self.top_embed = self.sd.get_text_embeds(top_prompt, "") + self.side_embed = self.sd.get_text_embeds(side_prompt, "") + self.back_embed = self.sd.get_text_embeds(back_prompt, "") + self.front_embed = self.sd.get_text_embeds(front_prompt, "") - elif prompting_type == "base": - text_embedding = base + def get_text_embedding(self, vertical_angle: TensorType[1], horizonal_angle: TensorType[1]): + """Get text embedding based on the position of the camera relative to the scene. + This trick is used in Dreamfusion (https://dreamfusion3d.github.io/). - return text_embedding + Args: + vertical_angle: vertical angle of the camera + horizonal_angle: horizonal angle of the camera + """ + + if self.positional_prompting == "discrete": + if vertical_angle < 40: + text_embedding = self.top_embed + elif 315 < horizonal_angle <= 45: + text_embedding = self.front_embed + elif 45 < horizonal_angle <= 135: + text_embedding = self.side_embed + elif 135 < horizonal_angle <= 225: + text_embedding = self.back_embed + else: # horizonal_angle > 225 and horizonal_angle <= 315: + text_embedding = self.side_embed + elif self.positional_prompting == "interpolated": + horiz = horizonal_angle.to(self.sd_device) + vert = max(vertical_angle.to(self.sd_device), 0) + + if 0 < horizonal_angle <= 90: + text_embedding = (horiz) * self.side_embed + (90 - horiz) * self.front_embed + elif 90 < horizonal_angle <= 180: + text_embedding = (horiz - 90) * self.back_embed + (180 - horiz) * self.side_embed + elif 180 < horizonal_angle <= 270: + text_embedding = (horiz - 180) * self.side_embed + (270 - horiz) * self.back_embed + else: # 270 < horizonal_angle <= 360: + text_embedding = (horiz - 270) * self.front_embed + (360 - horiz) * self.side_embed + + text_embedding = text_embedding / 90.0 + text_embedding = (vert * text_embedding + (90 - vert) * self.top_embed) / 90.0 + else: + text_embedding = self.base_embed + + return text_embedding diff --git a/nerfstudio/models/dreamfusion.py b/nerfstudio/models/dreamfusion.py index d3a658851..86e45e0e1 100644 --- a/nerfstudio/models/dreamfusion.py +++ b/nerfstudio/models/dreamfusion.py @@ -23,8 +23,8 @@ from typing import Dict, List, Optional, Tuple, Type import numpy as np import torch -from torch.cuda.amp.grad_scaler import GradScaler from torch.nn import Parameter +from typing_extensions import Literal from nerfstudio.cameras.rays import RayBundle from nerfstudio.engine.callbacks import ( @@ -36,7 +36,7 @@ from nerfstudio.field_components.field_heads import FieldHeadNames from nerfstudio.fields.density_fields import HashMLPDensityField from nerfstudio.fields.dreamfusion_field import DreamFusionField from nerfstudio.generative.stable_diffusion import StableDiffusion -from nerfstudio.generative.stable_diffusion_utils import get_text_embedding +from nerfstudio.generative.stable_diffusion_utils import PositionalTextEmbeddings from nerfstudio.model_components.losses import ( MSELoss, distortion_loss, @@ -77,7 +77,7 @@ class DreamFusionModelConfig(ModelConfig): """Randomizes light source per output.""" initialize_density: bool = True """Initialize density in center of scene.""" - init_density_strength: float = 0.1 + init_density_strength: float = 10 """Initial strength of center density""" sphere_collider: bool = True """Use spherical collider instead of box""" @@ -135,8 +135,8 @@ class DreamFusionModelConfig(ModelConfig): """enables location based prompting""" interpolated_prompting: bool = False """enables interpolated location prompting""" - prompting_type: str = "location_based" - """choose between location_based, interpolated, base""" + positional_prompting: Literal["discrete", "interpolated", "base"] = "discrete" + """ how to incorporate position into prompt""" top_prompt: str = ", overhead view" """appended to prompt for overhead view""" side_prompt: str = ", side view" @@ -177,7 +177,7 @@ class DreamFusionModel(Model): self.target_transmittance = config.target_transmittance_start self.grad_scaler = kwargs["grad_scaler"] - self.prompting_type = config.prompting_type + self.positional_prompting = config.positional_prompting self.guidance_scale = config.guidance_scale self.top_prompt = config.top_prompt self.side_prompt = config.side_prompt @@ -197,13 +197,15 @@ class DreamFusionModel(Model): super().populate_modules() self.sd = StableDiffusion(self.sd_device, version=self.sd_version) - self.text_embeddings = { - "top_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.top_prompt}", ""), - "front_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.front_prompt}", ""), - "side_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.side_prompt}", ""), - "back_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.back_prompt}", ""), - "base_text_embedding": self.sd.get_text_embeds(self.cur_prompt, ""), - } + self.text_embeddings = PositionalTextEmbeddings( + base_prompt=self.cur_prompt, + top_prompt=self.cur_prompt + self.top_prompt, + side_prompt=self.cur_prompt + self.side_prompt, + back_prompt=self.cur_prompt + self.back_prompt, + front_prompt=self.cur_prompt + self.front_prompt, + stable_diffusion=self.sd, + positional_prompting=self.positional_prompting, + ) # setting up fields self.field = DreamFusionField(self.scene_box.aabb, max_res=self.config.max_res) @@ -340,25 +342,26 @@ class DreamFusionModel(Model): param_groups["fields"] = list(self.field.parameters()) return param_groups - def get_outputs(self, ray_bundle: RayBundle): + def get_outputs(self, ray_bundle: RayBundle): # pylint: disable=too-many-statements # uniform sampling background_rgb = self.field.get_background_rgb(ray_bundle) ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns) field_outputs = self.field(ray_samples, compute_normals=True) - weights = ray_samples.get_weights(field_outputs[FieldHeadNames.DENSITY]) - weights_list.append(weights) - ray_samples_list.append(ray_samples) density = field_outputs[FieldHeadNames.DENSITY] - if self.initialize_density and self.training: + if self.initialize_density: pos = ray_samples.frustums.get_positions() density_blob = ( self.config.init_density_strength * self.density_strength - * torch.exp(-torch.norm(pos, dim=-1) / (2 * 0.04))[..., None] + * torch.exp(-torch.norm(pos, dim=-1) * 10)[..., None] ) density += density_blob + weights = ray_samples.get_weights(density) + weights_list.append(weights) + ray_samples_list.append(ray_samples) + accumulation = self.renderer_accumulation(weights) depth = self.renderer_depth(weights, ray_samples) rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights) @@ -422,7 +425,7 @@ class DreamFusionModel(Model): else: outputs["train_output"] = outputs["rgb"] - if self.training or True: + if self.training: outputs["rendered_orientation_loss"] = orientation_loss( weights.detach(), field_outputs[FieldHeadNames.NORMALS], ray_bundle.directions ) @@ -464,15 +467,17 @@ class DreamFusionModel(Model): if self.prompt != self.cur_prompt: self.cur_prompt = self.prompt - self.text_embeddings = { - "top_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.top_prompt}", ""), - "front_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.front_prompt}", ""), - "side_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.side_prompt}", ""), - "back_text_embedding": self.sd.get_text_embeds(f"{self.cur_prompt}{self.back_prompt}", ""), - "base_text_embedding": self.sd.get_text_embeds(self.cur_prompt, ""), - } + self.text_embeddings.update_prompt( + base_prompt=self.cur_prompt, + top_prompt=self.cur_prompt + self.top_prompt, + side_prompt=self.cur_prompt + self.side_prompt, + back_prompt=self.cur_prompt + self.back_prompt, + front_prompt=self.cur_prompt + self.front_prompt, + ) - text_embedding = get_text_embedding(batch, self.prompting_type, self.text_embeddings, self.sd_device) + text_embedding = self.text_embeddings.get_text_embedding( + vertical_angle=batch["vertical"], horizonal_angle=batch["central"] + ) train_output = ( outputs["train_output"] @@ -480,13 +485,12 @@ class DreamFusionModel(Model): .permute(0, 3, 1, 2) ) - with torch.autocast(device_type="cuda", dtype=torch.float16): - sds_loss = self.sd.sds_loss( - text_embedding.to(self.sd_device), - train_output.to(self.sd_device), - guidance_scale=int(self.guidance_scale), - grad_scaler=self.grad_scaler, - ) + sds_loss = self.sd.sds_loss( + text_embedding.to(self.sd_device), + train_output.to(self.sd_device), + guidance_scale=int(self.guidance_scale), + grad_scaler=self.grad_scaler, + ) loss_dict["sds_loss"] = sds_loss.to(self.device) diff --git a/nerfstudio/viewer/server/viewer_utils.py b/nerfstudio/viewer/server/viewer_utils.py index bbdd1099b..0abd1eaf3 100644 --- a/nerfstudio/viewer/server/viewer_utils.py +++ b/nerfstudio/viewer/server/viewer_utils.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=too-many-lines + """Code to interface with the `vis/` (the JS viewer). """ from __future__ import annotations From 293ae8d6430c579ecb4ffd718ee4561d4f61b22f Mon Sep 17 00:00:00 2001 From: Matt Tancik Date: Tue, 21 Feb 2023 09:51:46 -0800 Subject: [PATCH 17/17] Fix test --- nerfstudio/generative/stable_diffusion.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/nerfstudio/generative/stable_diffusion.py b/nerfstudio/generative/stable_diffusion.py index 51e03cecb..752fd95a3 100644 --- a/nerfstudio/generative/stable_diffusion.py +++ b/nerfstudio/generative/stable_diffusion.py @@ -33,19 +33,16 @@ from torch.cuda.amp import custom_bwd, custom_fwd from torch.cuda.amp.grad_scaler import GradScaler from torchtyping import TensorType -CONSOLE = Console(width=120) - try: from diffusers import PNDMScheduler, StableDiffusionPipeline from transformers import logging -except ImportError: - CONSOLE.print("[bold red]Missing Stable Diffusion packages.") - CONSOLE.print(r"Install using [yellow]pip install nerfstudio\[gen][/yellow]") - CONSOLE.print(r"or [yellow]pip install -e .\[gen][/yellow] if installing from source.") - sys.exit(1) + SD_AVAILABLE = True -logging.set_verbosity_error() +except ImportError: + SD_AVAILABLE = False + +CONSOLE = Console(width=120) IMG_DIM = 512 CONST_SCALE = 0.18215 SD_IDENTIFIERS = { @@ -92,6 +89,14 @@ class StableDiffusion(nn.Module): def __init__(self, device: Union[torch.device, str], num_train_timesteps: int = 1000, version="1-5") -> None: super().__init__() + if not SD_AVAILABLE: + CONSOLE.print("[bold red]Missing Stable Diffusion packages.") + CONSOLE.print(r"Install using [yellow]pip install nerfstudio\[gen][/yellow]") + CONSOLE.print(r"or [yellow]pip install -e .\[gen][/yellow] if installing from source.") + sys.exit(1) + + logging.set_verbosity_error() + self.device = device self.num_train_timesteps = num_train_timesteps