diff --git a/.vscode/launch.json b/.vscode/launch.json index 8b92b869d..935d7ae8b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -77,8 +77,8 @@ "--viewer.enable", "--viewer.zmq-url=tcp://127.0.0.1:6001", "--viewer.websocket-port=8014", - "--trainer.steps_per_save=1000000", - "--trainer.steps_per_test=1000000", + "--steps_per_save=1000000", + "--steps_per_test=1000000", "--pipeline.model.density-field-params.base-scale=4", "--pipeline.model.density-field-params.num-cascades=1", "pipeline.datamanager.train-dataparser:friends-data-parser-config", diff --git a/Dockerfile b/Dockerfile index 9cfcb77b6..c2e3ab82b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Define base image. -FROM nvidia/cudagl:11.3.1-devel +FROM nvidia/cuda:11.7.1-devel-ubuntu22.04 # Set environment variables. ## Set non-interactive to prevent asking for user inputs blocking image creation. @@ -11,7 +11,7 @@ ENV TCNN_CUDA_ARCHITECTURES=86 ## CUDA Home, required to find CUDA in some packages. ENV CUDA_HOME="/usr/local/cuda" -# Install required apt packages. +# Install required apt packages and clear cache afterwards. RUN apt-get update && \ apt-get install -y --no-install-recommends \ build-essential \ @@ -36,10 +36,11 @@ RUN apt-get update && \ libsuitesparse-dev \ nano \ protobuf-compiler \ - python3.8-dev \ + python3.10-dev \ python3-pip \ qtbase5-dev \ - wget + wget && \ + rm -rf /var/lib/apt/lists/* # Install GLOG (required by ceres). RUN git clone --branch v0.6.0 https://github.com/google/glog.git --single-branch && \ @@ -50,7 +51,7 @@ RUN git clone --branch v0.6.0 https://github.com/google/glog.git --single-branch make -j && \ make install && \ cd ../.. && \ - rm -r glog + rm -rf glog # Add glog path to LD_LIBRARY_PATH. ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" @@ -64,18 +65,19 @@ RUN git clone --branch 2.1.0 https://ceres-solver.googlesource.com/ceres-solver. make -j && \ make install && \ cd ../.. && \ - rm -r ceres-solver + rm -rf ceres-solver # Install colmap. RUN git clone --branch 3.7 https://github.com/colmap/colmap.git --single-branch && \ cd colmap && \ mkdir build && \ cd build && \ - cmake .. && \ + cmake .. -DCUDA_ENABLED=ON \ + -DCUDA_NVCC_FLAGS="--std c++14" && \ make -j && \ make install && \ cd ../.. && \ - rm -r colmap + rm -rf colmap # Create non root user and setup environment. RUN useradd -m -d /home/user -u 1000 user @@ -89,11 +91,11 @@ ENV PATH="${PATH}:/home/user/.local/bin" SHELL ["/bin/bash", "-c"] # Upgrade pip and install packages. -RUN python3.8 -m pip install --upgrade pip setuptools pathtools promise +RUN python3.10 -m pip install --upgrade pip setuptools pathtools promise # Install pytorch and submodules. -RUN python3.8 -m pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu113 +RUN python3.10 -m pip install torch==1.12.1+cu116 torchvision==0.13.1+cu116 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu116 # Install tynyCUDNN. -RUN python3.8 -m pip install git+https://github.com/NVlabs/tiny-cuda-nn.git#subdirectory=bindings/torch +RUN python3.10 -m pip install git+https://github.com/NVlabs/tiny-cuda-nn.git#subdirectory=bindings/torch # Copy nerfstudio folder and give ownership to user. ADD . /home/user/nerfstudio @@ -103,7 +105,7 @@ USER 1000:1000 # Install nerfstudio dependencies. RUN cd nerfstudio && \ - python3.8 -m pip install -e . && \ + python3.10 -m pip install -e . && \ cd .. # Change working directory diff --git a/README.md b/README.md index edef31622..923753098 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ Navigating to the link at the end of the terminal will load the webviewer. If yo It is possible to load a pretrained model by running ```bash -ns-train nerfacto --data data/nerfstudio/poster --trainer.load-dir {outputs/.../nerfstudio_models} +ns-train nerfacto --data data/nerfstudio/poster --load-dir {outputs/.../nerfstudio_models} ``` This will automatically start training. If you do not want it to train, add `--viewer.start-train False` to your training command. diff --git a/colab/demo.ipynb b/colab/demo.ipynb index 9609a6db4..10d0d601a 100644 --- a/colab/demo.ipynb +++ b/colab/demo.ipynb @@ -7,7 +7,7 @@ "colab_type": "text" }, "source": [ - "\"Open" + "\"Open" ] }, { @@ -70,18 +70,18 @@ "base_uri": "https://localhost:8080/" }, "id": "RGr33zHaHak0", - "outputId": "2641dce6-cd57-4d31-d8b1-860bc1adf17f" + "outputId": "6ad38334-22d2-4675-bffb-0bae91b17577" }, "outputs": [ { - "name": "stdout", "output_type": "stream", + "name": "stdout", "text": [ "⏬ Downloading https://github.com/jaimergp/miniforge/releases/latest/download/Mambaforge-colab-Linux-x86_64.sh...\n", "📦 Installing...\n", "📌 Adjusting configuration...\n", "🩹 Patching environment...\n", - "⏲ Done in 0:00:27\n", + "⏲ Done in 0:00:18\n", "🔁 Restarting kernel...\n" ] } @@ -111,9 +111,8 @@ "\n", "# Installing TinyCuda\n", "%cd /content/\n", - "# !gdown \"https://drive.google.com/u/1/uc?id=1wglEFPbI4BjWMFYaKk7gHCNsWCZphKjs\" \n", - "# !pip install tinycudann-1.6-cp38-cp38-linux_x86_64.whl\n", - "!pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch\n", + "!gdown \"https://drive.google.com/u/0/uc?id=1mB3IxcpmTVmQN-So0t2lxjxbHvqWr2zf\" \n", + "!pip install tinycudann-1.5-cp38-cp38-linux_x86_64.whl\n", "\n", "# Installing COLMAP\n", "%cd /content/\n", @@ -122,7 +121,8 @@ "# Install nerfstudio\n", "%cd /content/\n", "# !pip install nerfstudio\n", - "!pip install git+https://github.com/nerfstudio-project/nerfstudio.git" + "!pip install git+https://github.com/nerfstudio-project/nerfstudio.git\n", + "!conda remove --force qt-main" ] }, { diff --git a/docs/quickstart/first_nerf.md b/docs/quickstart/first_nerf.md index 1bd6ec367..30143814f 100644 --- a/docs/quickstart/first_nerf.md +++ b/docs/quickstart/first_nerf.md @@ -35,7 +35,7 @@ Navigating to the link at the end of the terminal will load the webviewer. If yo It is possible to load a pretrained model by running ```bash -ns-train nerfacto --data data/nerfstudio/poster --trainer.load-dir {outputs/.../nerfstudio_models} +ns-train nerfacto --data data/nerfstudio/poster --load-dir {outputs/.../nerfstudio_models} ``` This will automatically start training. If you do not want it to train, add `--viewer.start-train False` to your training command. diff --git a/docs/quickstart/installation.md b/docs/quickstart/installation.md index 62e95c228..cf08348c8 100644 --- a/docs/quickstart/installation.md +++ b/docs/quickstart/installation.md @@ -66,7 +66,7 @@ pip install -e .[docs] ## Use docker image Instead of installing and compiling prerequisites, setting up the environment and installing dependencies, a ready to use docker image is provided. \ ### Prerequisites -Docker ([get docker](https://docs.docker.com/get-docker/)) and nvidia GPU drivers ([get nvidia drivers](https://www.nvidia.de/Download/index.aspx?lang=de)), capable of working with CUDA 11.3, must be installed. +Docker ([get docker](https://docs.docker.com/get-docker/)) and nvidia GPU drivers ([get nvidia drivers](https://www.nvidia.de/Download/index.aspx?lang=de)), capable of working with CUDA 11.7, must be installed. The docker image can then either be pulled from [here](https://hub.docker.com/r/dromni/nerfstudio/tags) (replace with the actual version, e.g. 0.1.10) ```bash docker pull dromni/nerfstudio: @@ -89,15 +89,18 @@ docker run --gpus all \ # Give the conta ### Call nerfstudio commands directly Besides, the container can also directly be used by adding the nerfstudio command to the end. ```bash -docker run --gpus all -v /folder/of/your/data:/workspace/ -v /home//.cache/:/home/user/.cache/ -p 7007:7007 --rm # Parameters. +docker run --gpus all -v /folder/of/your/data:/workspace/ -v /home//.cache/:/home/user/.cache/ -p 7007:7007 --rm -it # Parameters. nerfstudio \ # Docker image name ns-process-data video --data /workspace/video.mp4 # Smaple command of nerfstudio. ``` ### Note - The container works on Linux and Windows, depending on your OS some additional setup steps might be required to provide access to your GPU inside containers. +- Paths on Windows use backslash '\\' while unix based systems use a frontslash '/' for paths, where backslashes might require an escape character depending on where they are used (e.g. C:\\\\folder1\\\\folder2...). Ensure to use the correct paths when mounting folders or providing paths as parameters. - Everything inside the container, what is not in a mounted folder (workspace in the above example), will be permanently removed after destroying the container. Always do all your tasks and output folder in workdir! -- The user inside the container is called user and is mapped to the local user with ID 1000 (usually the first non-root user on Linux systems). -- The container currently is based on nvidia:cudagl-11.3.1, consequently it comes with CUDA 11.3 which must be supported by the nvidia driver. No local CUDA installation is required or will be affected by using the docker image. +- The user inside the container is called 'user' and is mapped to the local user with ID 1000 (usually the first non-root user on Linux systems). +- The container currently is based on nvidia/cuda:11.7.1-devel-ubuntu22.04, consequently it comes with CUDA 11.7 which must be supported by the nvidia driver. No local CUDA installation is required or will be affected by using the docker image. +- The docker image (respectively Ubuntu 22.04) comes with Python3.10, no older version of Python is installed. +- If you call the container with commands directly, you still might want to add the interactive terminal ('-it') flag to get live log outputs of the nerfstudio scripts. In case the container is used in an automated environment the flag should be discarded. ## Installation FAQ diff --git a/docs/reference/contributing.md b/docs/reference/contributing.md index e622e22f9..45111c373 100644 --- a/docs/reference/contributing.md +++ b/docs/reference/contributing.md @@ -2,7 +2,7 @@ **💝 We're excited to have you join the nerfstudio family 💝** -Below, we document the contribution pipeline and good-to-knows for when you're ready to submit a PR. If you have any questions at all, please don't hesitate to reach out to us on [Discord](https://discord.com/invite/NHGtYRAW). We'd love to hear from you! +Below, we document the contribution pipeline and good-to-knows for when you're ready to submit a PR. If you have any questions at all, please don't hesitate to reach out to us on [Discord](https://discord.gg/uMbNqcraFc). We'd love to hear from you! The project is set up for development in VSCode, we recommend using it if you plan to contribute. @@ -35,8 +35,8 @@ conda install -c conda-forge pandoc ## Committing code -1. Make your modifications ✏️ -2. Perform local checks ✅ +1. Make your modifications ✏️ +2. Perform local checks ✅ To ensure that you will be passing all tests and checks on github, you will need to run the following command: @@ -50,13 +50,13 @@ conda install -c conda-forge pandoc - Pytests: Runs pytests locally to make sure added code does not break existing logic. - Documentation build: Builds docs locally. Ensures changes do not result in warnings/errors. - Licensing: Automatically adds licensing headers to the correct files. - + :::{admonition} Attention :class: attention In order to merge changes to the code base, all of these checks must be passing. If you pass these tests locally, you will likely pass on github servers as well (results in a green checkmark next to your commit). ::: -3. Open pull request! 💌 +3. Open pull request! 💌 ## Maintaining documentation diff --git a/nerfstudio/cameras/camera_optimizers.py b/nerfstudio/cameras/camera_optimizers.py index 891cd272a..b8d846f02 100644 --- a/nerfstudio/cameras/camera_optimizers.py +++ b/nerfstudio/cameras/camera_optimizers.py @@ -29,14 +29,14 @@ from torchtyping import TensorType from typing_extensions import Literal, assert_never from nerfstudio.cameras.lie_groups import exp_map_SE3, exp_map_SO3xR3 -from nerfstudio.configs import base_config as cfg +from nerfstudio.configs.base_config import InstantiateConfig from nerfstudio.engine.optimizers import AdamOptimizerConfig from nerfstudio.engine.schedulers import SchedulerConfig from nerfstudio.utils import poses as pose_utils @dataclass -class CameraOptimizerConfig(cfg.InstantiateConfig): +class CameraOptimizerConfig(InstantiateConfig): """Configuration of optimization for camera poses.""" _target: Type = field(default_factory=lambda: CameraOptimizer) diff --git a/nerfstudio/cameras/camera_paths.py b/nerfstudio/cameras/camera_paths.py index fb243917d..f5eb67501 100644 --- a/nerfstudio/cameras/camera_paths.py +++ b/nerfstudio/cameras/camera_paths.py @@ -23,7 +23,7 @@ import torch import nerfstudio.utils.poses as pose_utils from nerfstudio.cameras import camera_utils from nerfstudio.cameras.camera_utils import get_interpolated_poses_many -from nerfstudio.cameras.cameras import Cameras +from nerfstudio.cameras.cameras import Cameras, CameraType from nerfstudio.viewer.server.utils import three_js_perspective_camera_focal_length @@ -121,6 +121,19 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras: image_height = camera_path["render_height"] image_width = camera_path["render_width"] + if "camera_type" in camera_path: + camera_type = camera_path["camera_type"] + else: + camera_type = "perspective" + + if "camera_type" not in camera_path: + camera_type = CameraType.PERSPECTIVE + elif camera_path["camera_type"] == "fisheye": + camera_type = CameraType.FISHEYE + elif camera_path["camera_type"] == "equirectangular": + camera_type = CameraType.EQUIRECTANGULAR + else: + camera_type = CameraType.PERSPECTIVE c2ws = [] fxs = [] @@ -129,11 +142,15 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras: # pose c2w = torch.tensor(camera["camera_to_world"]).view(4, 4)[:3] c2ws.append(c2w) - # field of view - fov = camera["fov"] - focal_length = three_js_perspective_camera_focal_length(fov, image_height) - fxs.append(focal_length) - fys.append(focal_length) + if camera_type == CameraType.EQUIRECTANGULAR: + fxs.append(image_width / 2) + fys.append(image_height) + else: + # field of view + fov = camera["fov"] + focal_length = three_js_perspective_camera_focal_length(fov, image_height) + fxs.append(focal_length) + fys.append(focal_length) camera_to_worlds = torch.stack(c2ws, dim=0) fx = torch.tensor(fxs) @@ -144,4 +161,5 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras: cx=image_width / 2, cy=image_height / 2, camera_to_worlds=camera_to_worlds, + camera_type=camera_type, ) diff --git a/nerfstudio/cameras/cameras.py b/nerfstudio/cameras/cameras.py index 0ec8ed2d2..09b865344 100644 --- a/nerfstudio/cameras/cameras.py +++ b/nerfstudio/cameras/cameras.py @@ -634,7 +634,7 @@ class Cameras(TensorDataclass): directions_stack[..., 0][mask] = torch.masked_select(coord_stack[..., 0] * sin_theta / theta, mask).float() directions_stack[..., 1][mask] = torch.masked_select(coord_stack[..., 1] * sin_theta / theta, mask).float() - directions_stack[..., 2][mask] = -torch.masked_select(torch.cos(theta), mask) + directions_stack[..., 2][mask] = -torch.masked_select(torch.cos(theta), mask).float() if CameraType.EQUIRECTANGULAR.value in cam_types: mask = (self.camera_type[true_indices] == CameraType.EQUIRECTANGULAR.value).squeeze(-1) # (num_rays) diff --git a/nerfstudio/configs/base_config.py b/nerfstudio/configs/base_config.py index a74f07ed7..aa0177ce3 100644 --- a/nerfstudio/configs/base_config.py +++ b/nerfstudio/configs/base_config.py @@ -120,35 +120,6 @@ class LoggingConfig(PrintableConfig): profiler logs run times of functions and prints at end of training""" -# Trainer related configs -@dataclass -class TrainerConfig(PrintableConfig): - """Configuration for training regimen""" - - steps_per_save: int = 1000 - """Number of steps between saves.""" - steps_per_eval_batch: int = 500 - """Number of steps between randomly sampled batches of rays.""" - steps_per_eval_image: int = 500 - """Number of steps between single eval images.""" - steps_per_eval_all_images: int = 25000 - """Number of steps between eval all images.""" - max_num_iterations: int = 1000000 - """Maximum number of iterations to run.""" - mixed_precision: bool = False - """Whether or not to use mixed precision for training.""" - relative_model_dir: Path = Path("nerfstudio_models/") - """Relative path to save all checkpoints.""" - save_only_latest_checkpoint: bool = True - """Whether to only save the latest checkpoint or all checkpoints.""" - # optional parameters if we want to resume training - load_dir: Optional[Path] = None - """Optionally specify a pre-trained model directory to load from.""" - load_step: Optional[int] = None - """Optionally specify model step to load from; if none, will find most recent model in load_dir.""" - load_config: Optional[Path] = None - - # Viewer related configs @dataclass class ViewerConfig(PrintableConfig): diff --git a/nerfstudio/configs/experiment_config.py b/nerfstudio/configs/experiment_config.py index e78c8de59..16bbf129d 100644 --- a/nerfstudio/configs/experiment_config.py +++ b/nerfstudio/configs/experiment_config.py @@ -14,6 +14,8 @@ """Config used for running an experiment""" +from __future__ import annotations + from dataclasses import dataclass from datetime import datetime from pathlib import Path @@ -24,10 +26,9 @@ from rich.console import Console from typing_extensions import Literal from nerfstudio.configs.base_config import ( + InstantiateConfig, LoggingConfig, MachineConfig, - PrintableConfig, - TrainerConfig, ViewerConfig, ) from nerfstudio.configs.config_utils import to_immutable_dict @@ -39,8 +40,9 @@ CONSOLE = Console(width=120) @dataclass -class ExperimentConfig(PrintableConfig): - """Full config contents""" +class ExperimentConfig(InstantiateConfig): + """Full config contents for running an experiment. Any experiment types (like training) will be + subclassed from this, and must have their _target field defined accordingly.""" output_dir: Path = Path("outputs") """relative or absolute output directory to save all checkpoints and logging""" @@ -56,8 +58,6 @@ class ExperimentConfig(PrintableConfig): """Logging configuration""" viewer: ViewerConfig = ViewerConfig() """Viewer configuration""" - trainer: TrainerConfig = TrainerConfig() - """Trainer configuration""" pipeline: VanillaPipelineConfig = VanillaPipelineConfig() """Pipeline configuration""" optimizers: Dict[str, Any] = to_immutable_dict( @@ -73,6 +73,8 @@ class ExperimentConfig(PrintableConfig): """Which visualizer to use.""" data: Optional[Path] = None """Alias for --pipeline.datamanager.dataparser.data""" + relative_model_dir: Path = Path("nerfstudio_models/") + """Relative path to save all checkpoints.""" def is_viewer_enabled(self) -> bool: """Checks if a viewer is enabled.""" @@ -105,7 +107,7 @@ class ExperimentConfig(PrintableConfig): def get_checkpoint_dir(self) -> Path: """Retrieve the checkpoint directory""" - return Path(self.get_base_dir() / self.trainer.relative_model_dir) + return Path(self.get_base_dir() / self.relative_model_dir) def print_to_terminal(self) -> None: """Helper to pretty print config to terminal""" diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index 1fef60978..09c1cea01 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -24,8 +24,7 @@ import tyro from nerfacc import ContractionType from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig -from nerfstudio.configs.base_config import TrainerConfig, ViewerConfig -from nerfstudio.configs.experiment_config import ExperimentConfig +from nerfstudio.configs.base_config import ViewerConfig from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManagerConfig from nerfstudio.data.datamanagers.dreamfusion_datamanager import ( DreamFusionDataManagerConfig, @@ -46,6 +45,7 @@ from nerfstudio.data.dataparsers.phototourism_dataparser import ( ) from nerfstudio.engine.optimizers import AdamOptimizerConfig, RAdamOptimizerConfig from nerfstudio.engine.schedulers import SchedulerConfig +from nerfstudio.engine.trainer import TrainerConfig from nerfstudio.field_components.temporal_distortions import TemporalDistortionKind from nerfstudio.models.dreamfusion import DreamFusionModelConfig from nerfstudio.models.instant_ngp import InstantNGPModelConfig @@ -58,7 +58,7 @@ from nerfstudio.pipelines.base_pipeline import VanillaPipelineConfig from nerfstudio.pipelines.dreamfusion_pipeline import DreamfusionPipelineConfig from nerfstudio.pipelines.dynamic_batch import DynamicBatchPipelineConfig -method_configs: Dict[str, ExperimentConfig] = {} +method_configs: Dict[str, TrainerConfig] = {} descriptions = { "nerfacto": "Recommended real-time model tuned for real captures. This model will be continually updated.", "instant-ngp": "Implementation of Instant-NGP. Recommended real-time model for unbounded scenes.", @@ -72,11 +72,12 @@ descriptions = { "dreamfusion": "NeRF Model used in implementation of stable dreamfusion.", } -method_configs["nerfacto"] = ExperimentConfig( +method_configs["nerfacto"] = TrainerConfig( method_name="nerfacto", - trainer=TrainerConfig( - steps_per_eval_batch=500, steps_per_save=2000, max_num_iterations=30000, mixed_precision=True - ), + steps_per_eval_batch=500, + steps_per_save=2000, + max_num_iterations=30000, + mixed_precision=True, pipeline=VanillaPipelineConfig( datamanager=VanillaDataManagerConfig( dataparser=NerfstudioDataParserConfig(), @@ -102,11 +103,12 @@ method_configs["nerfacto"] = ExperimentConfig( vis="viewer", ) -method_configs["instant-ngp"] = ExperimentConfig( +method_configs["instant-ngp"] = TrainerConfig( method_name="instant-ngp", - trainer=TrainerConfig( - steps_per_eval_batch=500, steps_per_save=2000, max_num_iterations=30000, mixed_precision=True - ), + steps_per_eval_batch=500, + steps_per_save=2000, + max_num_iterations=30000, + mixed_precision=True, pipeline=DynamicBatchPipelineConfig( datamanager=VanillaDataManagerConfig(dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=8192), model=InstantNGPModelConfig(eval_num_rays_per_chunk=8192), @@ -122,11 +124,12 @@ method_configs["instant-ngp"] = ExperimentConfig( ) -method_configs["instant-ngp-bounded"] = ExperimentConfig( +method_configs["instant-ngp-bounded"] = TrainerConfig( method_name="instant-ngp-bounded", - trainer=TrainerConfig( - steps_per_eval_batch=500, steps_per_save=2000, max_num_iterations=30000, mixed_precision=True - ), + steps_per_eval_batch=500, + steps_per_save=2000, + max_num_iterations=30000, + mixed_precision=True, pipeline=DynamicBatchPipelineConfig( datamanager=VanillaDataManagerConfig(dataparser=InstantNGPDataParserConfig(), train_num_rays_per_batch=8192), model=InstantNGPModelConfig( @@ -149,7 +152,7 @@ method_configs["instant-ngp-bounded"] = ExperimentConfig( ) -method_configs["mipnerf"] = ExperimentConfig( +method_configs["mipnerf"] = TrainerConfig( method_name="mipnerf", pipeline=VanillaPipelineConfig( datamanager=VanillaDataManagerConfig(dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=1024), @@ -169,11 +172,12 @@ method_configs["mipnerf"] = ExperimentConfig( }, ) -method_configs["semantic-nerfw"] = ExperimentConfig( +method_configs["semantic-nerfw"] = TrainerConfig( method_name="semantic-nerfw", - trainer=TrainerConfig( - steps_per_eval_batch=500, steps_per_save=2000, max_num_iterations=30000, mixed_precision=True - ), + steps_per_eval_batch=500, + steps_per_save=2000, + max_num_iterations=30000, + mixed_precision=True, pipeline=VanillaPipelineConfig( datamanager=SemanticDataManagerConfig( dataparser=FriendsDataParserConfig(), train_num_rays_per_batch=4096, eval_num_rays_per_batch=8192 @@ -194,7 +198,7 @@ method_configs["semantic-nerfw"] = ExperimentConfig( vis="viewer", ) -method_configs["vanilla-nerf"] = ExperimentConfig( +method_configs["vanilla-nerf"] = TrainerConfig( method_name="vanilla-nerf", pipeline=VanillaPipelineConfig( datamanager=VanillaDataManagerConfig( @@ -214,9 +218,9 @@ method_configs["vanilla-nerf"] = ExperimentConfig( }, ) -method_configs["tensorf"] = ExperimentConfig( +method_configs["tensorf"] = TrainerConfig( method_name="tensorf", - trainer=TrainerConfig(mixed_precision=False), + mixed_precision=False, pipeline=VanillaPipelineConfig( datamanager=VanillaDataManagerConfig( dataparser=BlenderDataParserConfig(), @@ -235,7 +239,7 @@ method_configs["tensorf"] = ExperimentConfig( }, ) -method_configs["dnerf"] = ExperimentConfig( +method_configs["dnerf"] = TrainerConfig( method_name="dnerf", pipeline=VanillaPipelineConfig( datamanager=VanillaDataManagerConfig(dataparser=DNeRFDataParserConfig()), @@ -257,11 +261,12 @@ method_configs["dnerf"] = ExperimentConfig( }, ) -method_configs["phototourism"] = ExperimentConfig( +method_configs["phototourism"] = TrainerConfig( method_name="phototourism", - trainer=TrainerConfig( - steps_per_eval_batch=500, steps_per_save=2000, max_num_iterations=30000, mixed_precision=True - ), + steps_per_eval_batch=500, + steps_per_save=2000, + max_num_iterations=30000, + mixed_precision=True, pipeline=VanillaPipelineConfig( datamanager=VariableResDataManagerConfig( # NOTE: one of the only differences with nerfacto dataparser=PhototourismDataParserConfig(), # NOTE: one of the only differences with nerfacto diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index 140d018ba..26642a87c 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -42,6 +42,7 @@ from nerfstudio.data.dataparsers.friends_dataparser import FriendsDataParserConf from nerfstudio.data.dataparsers.instant_ngp_dataparser import ( InstantNGPDataParserConfig, ) +from nerfstudio.data.dataparsers.minimal_dataparser import MinimalDataParserConfig from nerfstudio.data.dataparsers.nerfstudio_dataparser import NerfstudioDataParserConfig from nerfstudio.data.dataparsers.nuscenes_dataparser import NuScenesDataParserConfig from nerfstudio.data.dataparsers.phototourism_dataparser import ( @@ -65,6 +66,7 @@ AnnotatedDataParserUnion = tyro.conf.OmitSubcommandPrefixes[ # Omit prefixes of tyro.extras.subcommand_type_from_defaults( { "nerfstudio-data": NerfstudioDataParserConfig(), + "minimal-parser": MinimalDataParserConfig(), "blender-data": BlenderDataParserConfig(), "friends-data": FriendsDataParserConfig(), "instant-ngp-data": InstantNGPDataParserConfig(), @@ -296,6 +298,8 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method train_dataset: InputDataset eval_dataset: InputDataset train_dataparser_outputs: DataparserOutputs + train_pixel_sampler: Optional[PixelSampler] = None + eval_pixel_sampler: Optional[PixelSampler] = None def __init__( self, @@ -406,6 +410,7 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method """Returns the next batch of data from the train dataloader.""" self.train_count += 1 image_batch = next(self.iter_train_image_dataloader) + assert self.train_pixel_sampler is not None batch = self.train_pixel_sampler.sample(image_batch) ray_indices = batch["indices"] ray_bundle = self.train_ray_generator(ray_indices) @@ -415,6 +420,7 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method """Returns the next batch of data from the eval dataloader.""" self.eval_count += 1 image_batch = next(self.iter_eval_image_dataloader) + assert self.eval_pixel_sampler is not None batch = self.eval_pixel_sampler.sample(image_batch) ray_indices = batch["indices"] ray_bundle = self.eval_ray_generator(ray_indices) diff --git a/nerfstudio/data/datamanagers/semantic_datamanager.py b/nerfstudio/data/datamanagers/semantic_datamanager.py index 188c7f6c3..322b1d3bd 100644 --- a/nerfstudio/data/datamanagers/semantic_datamanager.py +++ b/nerfstudio/data/datamanagers/semantic_datamanager.py @@ -41,6 +41,7 @@ class SemanticDataManager(VanillaDataManager): # pylint: disable=abstract-metho """ def create_train_dataset(self) -> SemanticDataset: + self.train_dataparser_outputs = self.dataparser.get_dataparser_outputs(split="train") return SemanticDataset( dataparser_outputs=self.train_dataparser_outputs, scale_factor=self.config.camera_res_scale_factor, diff --git a/nerfstudio/data/dataparsers/instant_ngp_dataparser.py b/nerfstudio/data/dataparsers/instant_ngp_dataparser.py index 6a3478749..d09aa86f9 100644 --- a/nerfstudio/data/dataparsers/instant_ngp_dataparser.py +++ b/nerfstudio/data/dataparsers/instant_ngp_dataparser.py @@ -46,8 +46,6 @@ class InstantNGPDataParserConfig(DataParserConfig): """target class to instantiate""" data: Path = Path("data/ours/posterv2") """Directory or explicit json file path specifying location of data.""" - scale_factor: float = 1.0 - """How much to scale the camera origins by.""" scene_scale: float = 0.3333 """How much to scale the scene.""" diff --git a/nerfstudio/data/dataparsers/minimal_dataparser.py b/nerfstudio/data/dataparsers/minimal_dataparser.py new file mode 100644 index 000000000..b6ffd8518 --- /dev/null +++ b/nerfstudio/data/dataparsers/minimal_dataparser.py @@ -0,0 +1,97 @@ +# Copyright 2022 The Nerfstudio Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Data parser for pre-prepared datasets for all cameras, with no additional processing needed +Optional fields - semantics, mask_filenames, cameras.distortion_params, cameras.times +""" + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Type + +import numpy as np +import torch + +from nerfstudio.cameras.cameras import Cameras +from nerfstudio.data.dataparsers.base_dataparser import ( + DataParser, + DataParserConfig, + DataparserOutputs, + Semantics, +) +from nerfstudio.data.scene_box import SceneBox + + +@dataclass +class MinimalDataParserConfig(DataParserConfig): + """Minimal dataset config""" + + _target: Type = field(default_factory=lambda: MinimalDataParser) + """target class to instantiate""" + data: Path = Path("/home/nikhil/nerfstudio-main/tests/data/lego_test/minimal_parser") + + +@dataclass +class MinimalDataParser(DataParser): + """Minimal DatasetParser""" + + config: MinimalDataParserConfig + + def _generate_dataparser_outputs(self, split="train"): + filepath = self.config.data / f"{split}.npz" + data = np.load(filepath, allow_pickle=True) + + image_filenames = data["image_filenames"].tolist() + mask_filenames = data["mask_filenames"].tolist() if "mask_filenames" in data.keys() else None + + metadata = None + if "semantics" in data.keys(): + semantics = data["semantics"].item() + metadata = { + "semantics": Semantics( + filenames=semantics["filenames"].tolist(), + classes=semantics["classes"].tolist(), + colors=semantics["colors"].tolist(), + mask_classes=semantics["mask_classes"].tolist(), + ) + } + + scene_box_aabb = torch.from_numpy(data["scene_box"]) + scene_box = SceneBox(aabb=scene_box_aabb) + + camera_np = data["cameras"].item() + distortion_params = None + if "distortion_params" in camera_np.keys(): + distortion_params = torch.from_numpy(camera_np["distortion_params"]) + cameras = Cameras( + fx=torch.from_numpy(camera_np["fx"]), + fy=torch.from_numpy(camera_np["fy"]), + cx=torch.from_numpy(camera_np["cx"]), + cy=torch.from_numpy(camera_np["cy"]), + distortion_params=distortion_params, + height=torch.from_numpy(camera_np["height"]), + width=torch.from_numpy(camera_np["width"]), + camera_to_worlds=torch.from_numpy(camera_np["camera_to_worlds"])[:, :3, :4], + camera_type=torch.from_numpy(camera_np["camera_type"]), + times=torch.from_numpy(camera_np["times"]) if "times" in camera_np.keys() else None, + ) + + dataparser_outputs = DataparserOutputs( + image_filenames=image_filenames, + cameras=cameras, + scene_box=scene_box, + mask_filenames=mask_filenames, + metadata=metadata, + ) + return dataparser_outputs diff --git a/nerfstudio/data/datasets/base_dataset.py b/nerfstudio/data/datasets/base_dataset.py index 1d1a88a3f..44793fbb0 100644 --- a/nerfstudio/data/datasets/base_dataset.py +++ b/nerfstudio/data/datasets/base_dataset.py @@ -64,7 +64,9 @@ class InputDataset(Dataset): width, height = pil_image.size newsize = (int(width * self.scale_factor), int(height * self.scale_factor)) pil_image = pil_image.resize(newsize, resample=Image.BILINEAR) - image = np.array(pil_image, dtype="uint8") # shape is (h, w, 3 or 4) + image = np.array(pil_image, dtype="uint8") # shape is (h, w) or (h, w, 3 or 4) + if len(image.shape) == 2: + image = image[:, :, None].repeat(3, axis=2) assert len(image.shape) == 3 assert image.dtype == np.uint8 assert image.shape[2] in [3, 4], f"Image shape of {image.shape} is in correct." diff --git a/nerfstudio/data/pixel_samplers.py b/nerfstudio/data/pixel_samplers.py index b76d39f16..8f0878568 100644 --- a/nerfstudio/data/pixel_samplers.py +++ b/nerfstudio/data/pixel_samplers.py @@ -17,120 +17,10 @@ Code for sampling pixels. """ import random -from typing import Dict +from typing import Dict, Optional, Union import torch - - -def collate_image_dataset_batch(batch: Dict, num_rays_per_batch: int, keep_full_image: bool = False): - """ - Operates on a batch of images and samples pixels to use for generating rays. - Returns a collated batch which is input to the Graph. - It will sample only within the valid 'mask' if it's specified. - - Args: - batch: batch of images to sample from - num_rays_per_batch: number of rays to sample per batch - keep_full_image: whether or not to include a reference to the full image in returned batch - """ - device = batch["image"].device - num_images, image_height, image_width, _ = batch["image"].shape - - # only sample within the mask, if the mask is in the batch - if "mask" in batch: - nonzero_indices = torch.nonzero(batch["mask"][..., 0], as_tuple=False) - chosen_indices = random.sample(range(len(nonzero_indices)), k=num_rays_per_batch) - indices = nonzero_indices[chosen_indices] - else: - indices = torch.floor( - torch.rand((num_rays_per_batch, 3), device=device) - * torch.tensor([num_images, image_height, image_width], device=device) - ).long() - - c, y, x = (i.flatten() for i in torch.split(indices, 1, dim=-1)) - collated_batch = {key: value[c, y, x] for key, value in batch.items() if key != "image_idx" and value is not None} - - assert collated_batch["image"].shape == (num_rays_per_batch, 3), collated_batch["image"].shape - - # Needed to correct the random indices to their actual camera idx locations. - indices[:, 0] = batch["image_idx"][c] - collated_batch["indices"] = indices # with the abs camera indices - - if keep_full_image: - collated_batch["full_image"] = batch["image"] - - return collated_batch - - -def collate_image_dataset_batch_list(batch: Dict, num_rays_per_batch: int, keep_full_image: bool = False): - """ - Does the same as collate_image_dataset_batch, except it will operate over a list of images / masks inside - a list. - - We will use this with the intent of DEPRECIATING it as soon as we find a viable alternative. - The intention will be to replace this with a more efficient implementation that doesn't require a for loop, but - since pytorch's ragged tensors are still in beta (this would allow for some vectorization), this will do - - Args: - batch: batch of images to sample from - num_rays_per_batch: number of rays to sample per batch - keep_full_image: whether or not to include a reference to the full image in returned batch - """ - - device = batch["image"][0].device - num_images = len(batch["image"]) - - # only sample within the mask, if the mask is in the batch - all_indices = [] - all_images = [] - - if "mask" in batch: - num_rays_in_batch = num_rays_per_batch // num_images - for i in range(num_images): - if i == num_images - 1: - num_rays_in_batch = num_rays_per_batch - (num_images - 1) * num_rays_in_batch - nonzero_indices = torch.nonzero(batch["mask"][i][..., 0], as_tuple=False) - chosen_indices = random.sample(range(len(nonzero_indices)), k=num_rays_in_batch) - indices = nonzero_indices[chosen_indices] - indices = torch.cat([torch.full((num_rays_in_batch, 1), i, device=device), indices], dim=-1) - all_indices.append(indices) - all_images.append(batch["image"][i][indices[:, 1], indices[:, 2]]) - - else: - num_rays_in_batch = num_rays_per_batch // num_images - for i in range(num_images): - image_height, image_width, _ = batch["image"][i].shape - if i == num_images - 1: - num_rays_in_batch = num_rays_per_batch - (num_images - 1) * num_rays_in_batch - indices = torch.floor( - torch.rand((num_rays_in_batch, 3), device=device) - * torch.tensor([1, image_height, image_width], device=device) - ).long() - indices[:, 0] = i - all_indices.append(indices) - all_images.append(batch["image"][i][indices[:, 1], indices[:, 2]]) - - indices = torch.cat(all_indices, dim=0) - - c, y, x = (i.flatten() for i in torch.split(indices, 1, dim=-1)) - collated_batch = { - key: value[c, y, x] - for key, value in batch.items() - if key != "image_idx" and key != "image" and key != "mask" and value is not None - } - - collated_batch["image"] = torch.cat(all_images, dim=0) - - assert collated_batch["image"].shape == (num_rays_per_batch, 3), collated_batch["image"].shape - - # Needed to correct the random indices to their actual camera idx locations. - indices[:, 0] = batch["image_idx"][c] - collated_batch["indices"] = indices # with the abs camera indices - - if keep_full_image: - collated_batch["full_image"] = batch["image"] - - return collated_batch +from torchtyping import TensorType class PixelSampler: # pylint: disable=too-few-public-methods @@ -141,7 +31,8 @@ class PixelSampler: # pylint: disable=too-few-public-methods keep_full_image: whether or not to include a reference to the full image in returned batch """ - def __init__(self, num_rays_per_batch: int, keep_full_image: bool = False) -> None: + def __init__(self, num_rays_per_batch: int, keep_full_image: bool = False, **kwargs) -> None: + self.kwargs = kwargs self.num_rays_per_batch = num_rays_per_batch self.keep_full_image = keep_full_image @@ -153,6 +44,144 @@ class PixelSampler: # pylint: disable=too-few-public-methods """ self.num_rays_per_batch = num_rays_per_batch + def sample_method( # pylint: disable=no-self-use + self, + batch_size: int, + num_images: int, + image_height: int, + image_width: int, + mask: Optional[TensorType] = None, + device: Union[torch.device, str] = "cpu", + ) -> TensorType["batch_size", 3]: + """ + Naive pixel sampler, uniformly samples across all possible pixels of all possible images. + + Args: + batch_size: number of samples in a batch + num_images: number of images to sample over + mask: mask of possible pixels in an image to sample from. + """ + if mask: + nonzero_indices = torch.nonzero(mask[..., 0], as_tuple=False) + chosen_indices = random.sample(range(len(nonzero_indices)), k=batch_size) + indices = nonzero_indices[chosen_indices] + else: + indices = torch.floor( + torch.rand((batch_size, 3), device=device) + * torch.tensor([num_images, image_height, image_width], device=device) + ).long() + + return indices + + def collate_image_dataset_batch(self, batch: Dict, num_rays_per_batch: int, keep_full_image: bool = False): + """ + Operates on a batch of images and samples pixels to use for generating rays. + Returns a collated batch which is input to the Graph. + It will sample only within the valid 'mask' if it's specified. + + Args: + batch: batch of images to sample from + num_rays_per_batch: number of rays to sample per batch + keep_full_image: whether or not to include a reference to the full image in returned batch + """ + + device = batch["image"].device + num_images, image_height, image_width, _ = batch["image"].shape + + if "mask" in batch: + indices = self.sample_method( + num_rays_per_batch, num_images, image_height, image_width, mask=batch["mask"], device=device + ) + else: + indices = self.sample_method(num_rays_per_batch, num_images, image_height, image_width, device=device) + + c, y, x = (i.flatten() for i in torch.split(indices, 1, dim=-1)) + collated_batch = { + key: value[c, y, x] for key, value in batch.items() if key != "image_idx" and value is not None + } + + assert collated_batch["image"].shape == (num_rays_per_batch, 3), collated_batch["image"].shape + + # Needed to correct the random indices to their actual camera idx locations. + indices[:, 0] = batch["image_idx"][c] + collated_batch["indices"] = indices # with the abs camera indices + + if keep_full_image: + collated_batch["full_image"] = batch["image"] + + return collated_batch + + def collate_image_dataset_batch_list(self, batch: Dict, num_rays_per_batch: int, keep_full_image: bool = False): + """ + Does the same as collate_image_dataset_batch, except it will operate over a list of images / masks inside + a list. + + We will use this with the intent of DEPRECIATING it as soon as we find a viable alternative. + The intention will be to replace this with a more efficient implementation that doesn't require a for loop, but + since pytorch's ragged tensors are still in beta (this would allow for some vectorization), this will do. + + Args: + batch: batch of images to sample from + num_rays_per_batch: number of rays to sample per batch + keep_full_image: whether or not to include a reference to the full image in returned batch + """ + + device = batch["image"][0].device + num_images = len(batch["image"]) + + # only sample within the mask, if the mask is in the batch + all_indices = [] + all_images = [] + + if "mask" in batch: + num_rays_in_batch = num_rays_per_batch // num_images + for i in range(num_images): + + image_height, image_width, _ = batch["image"][i].shape + + if i == num_images - 1: + num_rays_in_batch = num_rays_per_batch - (num_images - 1) * num_rays_in_batch + + indices = self.sample_method( + num_rays_in_batch, 1, image_height, image_width, mask=batch["mask"][i], device=device + ) + indices[:, 0] = i + all_indices.append(indices) + all_images.append(batch["image"][i][indices[:, 1], indices[:, 2]]) + + else: + num_rays_in_batch = num_rays_per_batch // num_images + for i in range(num_images): + image_height, image_width, _ = batch["image"][i].shape + if i == num_images - 1: + num_rays_in_batch = num_rays_per_batch - (num_images - 1) * num_rays_in_batch + indices = self.sample_method(num_rays_in_batch, 1, image_height, image_width, device=device) + indices[:, 0] = i + all_indices.append(indices) + all_images.append(batch["image"][i][indices[:, 1], indices[:, 2]]) + + indices = torch.cat(all_indices, dim=0) + + c, y, x = (i.flatten() for i in torch.split(indices, 1, dim=-1)) + collated_batch = { + key: value[c, y, x] + for key, value in batch.items() + if key != "image_idx" and key != "image" and key != "mask" and value is not None + } + + collated_batch["image"] = torch.cat(all_images, dim=0) + + assert collated_batch["image"].shape == (num_rays_per_batch, 3), collated_batch["image"].shape + + # Needed to correct the random indices to their actual camera idx locations. + indices[:, 0] = batch["image_idx"][c] + collated_batch["indices"] = indices # with the abs camera indices + + if keep_full_image: + collated_batch["full_image"] = batch["image"] + + return collated_batch + def sample(self, image_batch: Dict): """Sample an image batch and return a pixel batch. @@ -160,12 +189,12 @@ class PixelSampler: # pylint: disable=too-few-public-methods image_batch: batch of images to sample from """ if isinstance(image_batch["image"], list): - image_batch = dict(image_batch.items()) # copy the dictioary so we don't modify the original - pixel_batch = collate_image_dataset_batch_list( + image_batch = dict(image_batch.items()) # copy the dictionary so we don't modify the original + pixel_batch = self.collate_image_dataset_batch_list( image_batch, self.num_rays_per_batch, keep_full_image=self.keep_full_image ) elif isinstance(image_batch["image"], torch.Tensor): - pixel_batch = collate_image_dataset_batch( + pixel_batch = self.collate_image_dataset_batch( image_batch, self.num_rays_per_batch, keep_full_image=self.keep_full_image ) else: @@ -173,54 +202,6 @@ class PixelSampler: # pylint: disable=too-few-public-methods return pixel_batch -def collate_image_dataset_batch_equirectangular(batch: Dict, num_rays_per_batch: int, keep_full_image: bool = False): - """ - Operates on a batch of equirectangular images and samples pixels to use for - generating rays. Rays will be generated uniformly on the sphere. - Returns a collated batch which is input to the Graph. - It will sample only within the valid 'mask' if it's specified. - - Args: - batch: batch of images to sample from - num_rays_per_batch: number of rays to sample per batch - keep_full_image: whether or not to include a reference to the full image in returned batch - """ - # TODO(kevinddchen): make more DRY - device = batch["image"].device - num_images, image_height, image_width, _ = batch["image"].shape - - # only sample within the mask, if the mask is in the batch - if "mask" in batch: - # TODO(kevinddchen): implement this - raise NotImplementedError("Masking not implemented for equirectangular images.") - - # We sample theta uniformly in [0, 2*pi] - # We sample phi in [0, pi] according to the PDF f(phi) = sin(phi) / 2. - # This is done by inverse transform sampling. - # http://corysimon.github.io/articles/uniformdistn-on-sphere/ - num_images_rand = torch.rand(num_rays_per_batch, device=device) - phi_rand = torch.acos(1 - 2 * torch.rand(num_rays_per_batch, device=device)) / torch.pi - theta_rand = torch.rand(num_rays_per_batch, device=device) - indices = torch.floor( - torch.stack((num_images_rand, phi_rand, theta_rand), dim=-1) - * torch.tensor([num_images, image_height, image_width], device=device) - ).long() - - c, y, x = (i.flatten() for i in torch.split(indices, 1, dim=-1)) - collated_batch = {key: value[c, y, x] for key, value in batch.items() if key != "image_idx" and value is not None} - - assert collated_batch["image"].shape == (num_rays_per_batch, 3), collated_batch["image"].shape - - # Needed to correct the random indices to their actual camera idx locations. - indices[:, 0] = batch["image_idx"][c] - collated_batch["indices"] = indices # with the abs camera indices - - if keep_full_image: - collated_batch["full_image"] = batch["image"] - - return collated_batch - - class EquirectangularPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods """Samples 'pixel_batch's from 'image_batch's. Assumes images are equirectangular and the sampling is done uniformly on the sphere. @@ -231,9 +212,34 @@ class EquirectangularPixelSampler(PixelSampler): # pylint: disable=too-few-publ """ # overrides base method - def sample(self, image_batch: Dict): + def sample_method( # pylint: disable=no-self-use + self, + batch_size: int, + num_images: int, + image_height: int, + image_width: int, + mask: Optional[TensorType] = None, + device: Union[torch.device, str] = "cpu", + ) -> TensorType["batch_size", 3]: - pixel_batch = collate_image_dataset_batch_equirectangular( - image_batch, self.num_rays_per_batch, keep_full_image=self.keep_full_image - ) - return pixel_batch + if mask: + # Note: if there is a mask, sampling reduces back to uniform sampling, which gives more + # sampling weight to the poles of the image than the equators. + # TODO(kevinddchen): implement the correct mask-sampling method. + + indices = super().sample_method(batch_size, num_images, image_height, image_width, mask=mask, device=device) + else: + + # We sample theta uniformly in [0, 2*pi] + # We sample phi in [0, pi] according to the PDF f(phi) = sin(phi) / 2. + # This is done by inverse transform sampling. + # http://corysimon.github.io/articles/uniformdistn-on-sphere/ + num_images_rand = torch.rand(batch_size, device=device) + phi_rand = torch.acos(1 - 2 * torch.rand(batch_size, device=device)) / torch.pi + theta_rand = torch.rand(batch_size, device=device) + indices = torch.floor( + torch.stack((num_images_rand, phi_rand, theta_rand), dim=-1) + * torch.tensor([num_images, image_height, image_width], device=device) + ).long() + + return indices diff --git a/nerfstudio/engine/callbacks.py b/nerfstudio/engine/callbacks.py index 8f592669f..25f73d8d2 100644 --- a/nerfstudio/engine/callbacks.py +++ b/nerfstudio/engine/callbacks.py @@ -15,14 +15,13 @@ """ Callback code used for training iterations """ +from __future__ import annotations from dataclasses import InitVar, dataclass from enum import Enum, auto from inspect import signature from typing import Callable, Dict, List, Optional, Tuple -from nerfstudio.configs.base_config import TrainerConfig - @dataclass class TrainingCallbackAttributes: @@ -38,8 +37,6 @@ class TrainingCallbackAttributes: """gradient scalers""" pipeline: Optional[InitVar] """reference to training pipeline""" - config: TrainerConfig - """the trainer config""" class TrainingCallbackLocation(Enum): diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index e3bd1df80..48e6415e6 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -21,7 +21,9 @@ import dataclasses import functools import os import time -from typing import Dict, List, Tuple +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, List, Optional, Tuple, Type import torch from rich.console import Console @@ -49,6 +51,34 @@ from nerfstudio.viewer.server import viewer_utils CONSOLE = Console(width=120) +@dataclass +class TrainerConfig(ExperimentConfig): + """Configuration for training regimen""" + + _target: Type = field(default_factory=lambda: Trainer) + """target class to instantiate""" + steps_per_save: int = 1000 + """Number of steps between saves.""" + steps_per_eval_batch: int = 500 + """Number of steps between randomly sampled batches of rays.""" + steps_per_eval_image: int = 500 + """Number of steps between single eval images.""" + steps_per_eval_all_images: int = 25000 + """Number of steps between eval all images.""" + max_num_iterations: int = 1000000 + """Maximum number of iterations to run.""" + mixed_precision: bool = False + """Whether or not to use mixed precision for training.""" + save_only_latest_checkpoint: bool = True + """Whether to only save the latest checkpoint or all checkpoints.""" + # optional parameters if we want to resume training + load_dir: Optional[Path] = None + """Optionally specify a pre-trained model directory to load from.""" + load_step: Optional[int] = None + """Optionally specify model step to load from; if none, will find most recent model in load_dir.""" + load_config: Optional[Path] = None + + class Trainer: """Trainer class @@ -71,12 +101,12 @@ class Trainer: optimizers: Optimizers callbacks: List[TrainingCallback] - def __init__(self, config: ExperimentConfig, local_rank: int = 0, world_size: int = 1): + def __init__(self, config: TrainerConfig, local_rank: int = 0, world_size: int = 1): self.config = config self.local_rank = local_rank self.world_size = world_size self.device = "cpu" if world_size == 0 else f"cuda:{local_rank}" - self.mixed_precision = self.config.trainer.mixed_precision + self.mixed_precision = self.config.mixed_precision if self.device == "cpu": self.mixed_precision = False CONSOLE.print("Mixed precision is disabled for CPU training.") @@ -97,9 +127,7 @@ class Trainer: # set up writers/profilers if enabled writer_log_path = self.base_dir / config.logging.relative_log_dir writer.setup_event_writer(config.is_wandb_enabled(), config.is_tensorboard_enabled(), log_dir=writer_log_path) - writer.setup_local_writer( - config.logging, max_iter=config.trainer.max_num_iterations, banner_messages=banner_messages - ) + writer.setup_local_writer(config.logging, max_iter=config.max_num_iterations, banner_messages=banner_messages) writer.put_config(name="config", config_dict=dataclasses.asdict(config), step=0) profiler.setup_profiler(config.logging) @@ -124,7 +152,6 @@ class Trainer: optimizers=self.optimizers, # type: ignore grad_scaler=self.grad_scaler, # type: ignore pipeline=self.pipeline, # type: ignore - config=self.config.trainer, # type: ignore ) ) @@ -132,7 +159,7 @@ class Trainer: """Helper to set up the optimizers Returns: - The optimizers object given the experiment config. + The optimizers object given the trainer config. """ optimizer_config = self.config.optimizers.copy() camera_optimizer_config = self.config.pipeline.datamanager.camera_optimizer @@ -156,7 +183,7 @@ class Trainer: self._init_viewer_state() with TimeWriter(writer, EventName.TOTAL_TRAIN_TIME): - num_iterations = self.config.trainer.max_num_iterations + num_iterations = self.config.max_num_iterations step = 0 for step in range(self._start_step, self._start_step + num_iterations): with TimeWriter(writer, EventName.ITER_TRAIN_TIME, step=step) as train_t: @@ -195,7 +222,7 @@ class Trainer: self.eval_iteration(step) - if step_check(step, self.config.trainer.steps_per_save): + if step_check(step, self.config.steps_per_save): self.save_checkpoint(step) writer.write_out_storage() @@ -275,9 +302,9 @@ class Trainer: def _load_checkpoint(self) -> None: """Helper function to load pipeline and optimizer from prespecified checkpoint""" - load_dir = self.config.trainer.load_dir + load_dir = self.config.load_dir if load_dir is not None: - load_step = self.config.trainer.load_step + load_step = self.config.load_step if load_step is None: print("Loading latest checkpoint from load_dir") # NOTE: this is specific to the checkpoint name format @@ -318,7 +345,7 @@ class Trainer: ckpt_path, ) # possibly delete old checkpoints - if self.config.trainer.save_only_latest_checkpoint: + if self.config.save_only_latest_checkpoint: # delete everything else in the checkpoint folder for f in self.checkpoint_dir.glob("*"): if f != ckpt_path: @@ -360,7 +387,7 @@ class Trainer: step: Current training step. """ # a batch of eval rays - if step_check(step, self.config.trainer.steps_per_eval_batch): + if step_check(step, self.config.steps_per_eval_batch): _, eval_loss_dict, eval_metrics_dict = self.pipeline.get_eval_loss_dict(step=step) eval_loss = functools.reduce(torch.add, eval_loss_dict.values()) writer.put_scalar(name="Eval Loss", scalar=eval_loss, step=step) @@ -368,7 +395,7 @@ class Trainer: writer.put_dict(name="Eval Metrics Dict", scalar_dict=eval_metrics_dict, step=step) # one eval image - if step_check(step, self.config.trainer.steps_per_eval_image): + if step_check(step, self.config.steps_per_eval_image): with TimeWriter(writer, EventName.TEST_RAYS_PER_SEC, write=False) as test_t: metrics_dict, images_dict = self.pipeline.get_eval_image_metrics_and_images(step=step) writer.put_time( @@ -383,6 +410,6 @@ class Trainer: writer.put_image(name=group + "/" + image_name, image=image, step=step) # all eval images - if step_check(step, self.config.trainer.steps_per_eval_all_images): + if step_check(step, self.config.steps_per_eval_all_images): metrics_dict = self.pipeline.get_average_eval_image_metrics(step=step) writer.put_dict(name="Eval Images Metrics Dict (all images)", scalar_dict=metrics_dict, step=step) diff --git a/nerfstudio/field_components/spatial_distortions.py b/nerfstudio/field_components/spatial_distortions.py index 449f4b5b9..02da548a3 100644 --- a/nerfstudio/field_components/spatial_distortions.py +++ b/nerfstudio/field_components/spatial_distortions.py @@ -65,12 +65,8 @@ class SceneContraction(SpatialDistortion): def forward(self, positions): def contract(x): - mag = torch.linalg.norm(x, ord=self.order, dim=-1) - mask = mag >= 1 - x_new = x.clone() - x_new[mask] = (2 - (1 / mag[mask][..., None])) * (x[mask] / mag[mask][..., None]) - - return x_new + mag = torch.linalg.norm(x, ord=self.order, dim=-1)[..., None] + return torch.where(mag < 1, x, (2 - (1 / mag)) * (x / mag)) if isinstance(positions, Gaussians): means = contract(positions.mean.clone()) diff --git a/nerfstudio/fields/nerfacto_field.py b/nerfstudio/fields/nerfacto_field.py index 95ed66c1e..1c3865fee 100644 --- a/nerfstudio/fields/nerfacto_field.py +++ b/nerfstudio/fields/nerfacto_field.py @@ -69,6 +69,7 @@ class TCNNNerfactoField(Field): Args: aabb: parameters of scene aabb bounds + num_images: number of images in the dataset num_layers: number of hidden layers hidden_dim: dimension of hidden layers geo_feat_dim: output geo feat dimensions @@ -76,9 +77,15 @@ class TCNNNerfactoField(Field): max_res: maximum resolution of the hashmap for the base mlp log2_hashmap_size: size of the hashmap for the base mlp num_layers_color: number of hidden layers for color network + num_layers_transient: number of hidden layers for transient network hidden_dim_color: dimension of hidden layers for color network + hidden_dim_transient: dimension of hidden layers for transient network appearance_embedding_dim: dimension of appearance embedding transient_embedding_dim: dimension of transient embedding + use_transient_embedding: whether to use transient embedding + use_semantics: whether to use semantic segmentation + num_semantic_classes: number of semantic classes + use_pred_normals: whether to use predicted normals use_average_appearance_embedding: whether to use average appearance embedding or zeros for inference spatial_distortion: spatial distortion to apply to the scene """ diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py index 76634360a..6d43b227a 100644 --- a/nerfstudio/models/nerfacto.py +++ b/nerfstudio/models/nerfacto.py @@ -67,7 +67,7 @@ class NerfactoModelConfig(ModelConfig): """How far along the ray to start sampling.""" far_plane: float = 1000.0 """How far along the ray to stop sampling.""" - background_color: Literal["background", "last_sample"] = "last_sample" + background_color: Literal["random", "last_sample"] = "last_sample" """Whether to randomize the background color.""" num_levels: int = 16 """Number of levels of the hashmap for the base mlp.""" @@ -75,8 +75,8 @@ class NerfactoModelConfig(ModelConfig): """Maximum resolution of the hashmap for the base mlp.""" log2_hashmap_size: int = 19 """Size of the hashmap for the base mlp""" - num_proposal_samples_per_ray: Tuple[int] = (256, 96) - """Number of samples per ray for the proposal network.""" + num_proposal_samples_per_ray: Tuple[int, ...] = (256, 96) + """Number of samples per ray for each proposal network.""" num_nerf_samples_per_ray: int = 48 """Number of samples per ray for the nerf network.""" proposal_update_every: int = 5 diff --git a/nerfstudio/pipelines/dynamic_batch.py b/nerfstudio/pipelines/dynamic_batch.py index 788f68e82..293ea1706 100644 --- a/nerfstudio/pipelines/dynamic_batch.py +++ b/nerfstudio/pipelines/dynamic_batch.py @@ -64,8 +64,10 @@ class DynamicBatchPipeline(VanillaPipeline): def _update_pixel_samplers(self): """Update the pixel samplers for train and eval with the dynamic number of rays per batch.""" - self.datamanager.train_pixel_sampler.set_num_rays_per_batch(self.dynamic_num_rays_per_batch) - self.datamanager.eval_pixel_sampler.set_num_rays_per_batch(self.dynamic_num_rays_per_batch) + if self.datamanager.train_pixel_sampler is not None: + self.datamanager.train_pixel_sampler.set_num_rays_per_batch(self.dynamic_num_rays_per_batch) + if self.datamanager.eval_pixel_sampler is not None: + self.datamanager.eval_pixel_sampler.set_num_rays_per_batch(self.dynamic_num_rays_per_batch) def _update_dynamic_num_rays_per_batch(self, num_samples_per_batch: int): """Updates the dynamic number of rays per batch variable, diff --git a/nerfstudio/process_data/process_data_utils.py b/nerfstudio/process_data/process_data_utils.py index e028bf1c0..907a0af98 100644 --- a/nerfstudio/process_data/process_data_utils.py +++ b/nerfstudio/process_data/process_data_utils.py @@ -94,6 +94,7 @@ def convert_video_to_images( ffmpeg_cmd += f" -vf thumbnail={spacing},setpts=N/TB -r 1" else: CONSOLE.print("[bold red]Can't satify requested number of frames. Extracting all frames.") + ffmpeg_cmd += " -pix_fmt bgr8" ffmpeg_cmd += f" {out_filename}" diff --git a/nerfstudio/utils/eval_utils.py b/nerfstudio/utils/eval_utils.py index 8d780cc82..40f9e212e 100644 --- a/nerfstudio/utils/eval_utils.py +++ b/nerfstudio/utils/eval_utils.py @@ -27,8 +27,7 @@ import yaml from rich.console import Console from typing_extensions import Literal -from nerfstudio.configs.base_config import TrainerConfig -from nerfstudio.configs.experiment_config import ExperimentConfig +from nerfstudio.engine.trainer import TrainerConfig from nerfstudio.pipelines.base_pipeline import Pipeline CONSOLE = Console(width=120) @@ -69,7 +68,7 @@ def eval_setup( config_path: Path, eval_num_rays_per_chunk: Optional[int] = None, test_mode: Literal["test", "val", "inference"] = "test", -) -> Tuple[ExperimentConfig, Pipeline, Path]: +) -> Tuple[TrainerConfig, Pipeline, Path]: """Shared setup for loading a saved pipeline for evaluation. Args: @@ -86,14 +85,14 @@ def eval_setup( """ # load save config config = yaml.load(config_path.read_text(), Loader=yaml.Loader) - assert isinstance(config, ExperimentConfig) + assert isinstance(config, TrainerConfig) if eval_num_rays_per_chunk: config.pipeline.model.eval_num_rays_per_chunk = eval_num_rays_per_chunk # load checkpoints from wherever they were saved # TODO: expose the ability to choose an arbitrary checkpoint - config.trainer.load_dir = config.get_checkpoint_dir() + config.load_dir = config.get_checkpoint_dir() config.pipeline.datamanager.eval_image_indices = None # setup pipeline (which includes the DataManager) @@ -103,6 +102,6 @@ def eval_setup( pipeline.eval() # load checkpointed information - checkpoint_path = eval_load_checkpoint(config.trainer, pipeline) + checkpoint_path = eval_load_checkpoint(config, pipeline) return config, pipeline, checkpoint_path diff --git a/nerfstudio/viewer/app/package.json b/nerfstudio/viewer/app/package.json index b6183729e..78d4997f3 100644 --- a/nerfstudio/viewer/app/package.json +++ b/nerfstudio/viewer/app/package.json @@ -1,7 +1,7 @@ { "name": "viewer", "homepage": ".", - "version": "22-12-02-0", + "version": "22-12-16-0", "private": true, "dependencies": { "@emotion/react": "^11.10.4", diff --git a/nerfstudio/viewer/app/src/index.scss b/nerfstudio/viewer/app/src/index.scss index 194c5ac2e..fab8b8a68 100644 --- a/nerfstudio/viewer/app/src/index.scss +++ b/nerfstudio/viewer/app/src/index.scss @@ -137,7 +137,10 @@ $sidebar-width: 100% !default; } .StatusPanel .StatusPanel-hide-scene-button { - margin: 12px; + margin-top: 4px; + margin-bottom: 4px; + margin-left: 6px; + margin-right: 6px; } .StatusPanel .WebSocketUrlField { @@ -242,18 +245,37 @@ $sidebar-width: 100% !default; // ------------------------------------ // Camera Pane -.CameraPanel { +.CameraPanel-props { + margin-top: 10px; + margin-bottom: 10px; + margin-left: 20px; + margin-right: 20px; +} + +.CameraPanel-path-row { + display: inline-block; + width: 45%; + margin-top: 8px; + margin-bottom: 8px; + margin-left: 4px; + margin-right: 4px; +} + +.CameraPanel-row { + display: inline-block; + margin-top: 8px; + margin-bottom: 8px; + margin-left: 4px; + margin-right: 4px; } .CameraPanel-top-button { - display: inline-block; - margin: 4px; - margin-top: 8px; + width: 100%; } .CameraPanel-render-button { display: inline-block; - width: 70%; + width: 92%; margin: 4px; margin-top: 8px; } diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx index 6a741d156..4efab1b6f 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx @@ -37,6 +37,7 @@ import { useDispatch, useSelector } from 'react-redux'; import { CSS2DObject } from 'three/examples/jsm/renderers/CSS2DRenderer'; import AddAPhotoIcon from '@mui/icons-material/AddAPhoto'; import FileDownloadOutlinedIcon from '@mui/icons-material/FileDownloadOutlined'; +import { LevaPanel, LevaStoreProvider, useCreateStore } from 'leva'; import FileUploadOutlinedIcon from '@mui/icons-material/FileUploadOutlined'; import { Stack } from '@mui/system'; import TextField from '@mui/material/TextField'; @@ -46,6 +47,8 @@ import { CameraHelper } from './CameraHelper'; import { get_curve_object_from_cameras, get_transform_matrix } from './curve'; import { WebSocketContext } from '../../WebSocket/WebSocket'; import RenderModal from '../../RenderModal'; +import CameraPropPanel from './CameraPropPanel'; +import LevaTheme from '../../../themes/leva_theme.json'; const msgpack = require('msgpack-lite'); @@ -109,12 +112,12 @@ function FovSelector(props) { camera.setFocalLength(val / camera.aspect); } } else if (fovLabel === FOV_LABELS.FOV) { + camera.fov = val; setGlobalFov(val); } else { - const old_fov = camera.fov; camera.setFocalLength(val / camera.aspect); const new_fov = camera.getEffectiveFOV(); - camera.fov = old_fov; + camera.fov = new_fov; setGlobalFov(new_fov); } @@ -213,6 +216,8 @@ function CameraList(props) { const [slider_value, set_slider_value] = React.useState(0); const [expanded, setExpanded] = React.useState(null); + const camera_type = useSelector((state) => state.renderingState.camera_type); + const handleChange = (cameraUUID: string) => (event: React.SyntheticEvent, isExpanded: boolean) => { @@ -396,7 +401,7 @@ function CameraList(props) { - {isAnimated('FOV') && ( + {isAnimated('FOV') && camera_type !== 'equirectangular' && ( state.renderingState.render_width, ); + const camera_type = useSelector((state) => state.renderingState.camera_type); const setRenderHeight = (value) => { dispatch({ @@ -495,11 +504,23 @@ export default function CameraPanel(props) { }); }; + const setCameraType = (value) => { + dispatch({ + type: 'write', + path: 'renderingState/camera_type', + data: value, + }); + }; + + const setFieldOfView = (value) => { + dispatch({ + type: 'write', + path: 'renderingState/field_of_view', + data: value, + }); + }; + // ui state - const [ui_render_height, setUIRenderHeight] = React.useState(render_height); - const [ui_render_width, setUIRenderWidth] = React.useState(render_width); - const [ui_seconds, setUISeconds] = React.useState(seconds); - const [ui_fps, setUIfps] = React.useState(fps); const [fovLabel, setFovLabel] = React.useState(FOV_LABELS.FOV); // nonlinear render option @@ -514,7 +535,7 @@ export default function CameraPanel(props) { // set slider and render camera back to 0 if (new_camera_list.length >= 1) { set_camera_position(camera_render, new_camera_list[0].matrix); - camera_render.fov = new_camera_list[0].fov; + setFieldOfView(new_camera_list[0].fov); set_slider_value(slider_min); } }; @@ -716,7 +737,7 @@ export default function CameraPanel(props) { const mat = get_transform_matrix(position, lookat, up); set_camera_position(camera_render, mat); - camera_render.fov = fov; + setFieldOfView(fov); } else { sceneTree.delete(['Camera Path', 'Curve']); } @@ -766,7 +787,7 @@ export default function CameraPanel(props) { fov = curve_object.curve_fovs.getPoint(point).z; const mat = get_transform_matrix(position, lookat, up); set_camera_position(camera_render, mat); - camera_render.fov = fov; + setFieldOfView(fov); setGlobalFov(fov); } }, [slider_value, render_height, render_width]); @@ -826,6 +847,7 @@ export default function CameraPanel(props) { // const myData const camera_path_object = { keyframes, + camera_type, render_height, render_width, camera_path, @@ -868,15 +890,11 @@ export default function CameraPanel(props) { const new_properties = new Map(cameraProperties); setRenderHeight(camera_path_object.render_height); - setUIRenderHeight(camera_path_object.render_height); setRenderWidth(camera_path_object.render_width); - setUIRenderWidth(camera_path_object.render_width); + setCameraType(camera_path_object.camera_type); setFps(camera_path_object.fps); - setUIfps(camera_path_object.fps); - setSeconds(camera_path_object.seconds); - setUISeconds(camera_path_object.seconds); set_smoothness_value(camera_path_object.smoothness_value); setIsCycle(camera_path_object.is_cycle); @@ -968,7 +986,7 @@ export default function CameraPanel(props) { return (
-
+
-
+
-
- { - if (e.target.validity.valid) { - setUIRenderHeight(e.target.value); - } - }} - onBlur={(e) => { - if (e.target.validity.valid) { - if (e.target.value !== '') { - setRenderHeight(parseInt(e.target.value, 10)); - } else { - setUIRenderHeight(render_height); - } - } - }} - value={ui_render_height} - error={ui_render_height <= 0} - helperText={ui_render_height <= 0 ? 'Required' : ''} - variant="standard" - /> - { - if (e.target.validity.valid) { - setUIRenderWidth(e.target.value); - } - }} - onBlur={(e) => { - if (e.target.validity.valid) { - if (e.target.value !== '') { - setRenderWidth(parseInt(e.target.value, 10)); - } else { - setUIRenderWidth(render_width); - } - } - }} - value={ui_render_width} - error={ui_render_width <= 0} - helperText={ui_render_width <= 0 ? 'Required' : ''} - variant="standard" +
+ + + +
-
- { - if (e.target.validity.valid) { - setUISeconds(e.target.value); - } - }} - onBlur={(e) => { - if (e.target.validity.valid) { - if (e.target.value !== '') { - setSeconds(parseInt(e.target.value, 10)); - } else { - setUISeconds(seconds); - } - } - }} - value={ui_seconds} - error={ui_seconds <= 0} - helperText={ui_seconds <= 0 ? 'Required' : ''} - variant="standard" - /> - { - if (e.target.validity.valid) { - setUIfps(e.target.value); - } - }} - onBlur={(e) => { - if (e.target.validity.valid) { - if (e.target.value !== '') { - setFps(parseInt(e.target.value, 10)); - } else { - setUIfps(fps); - } - } - }} - value={ui_fps} - error={ui_fps <= 0} - helperText={ui_fps <= 0 ? 'Required' : ''} - variant="standard" - /> -
-
- - - - -
+ sx={{ + mt: 1, + }} + > + + + + +
+ )}
-
+
-
+
-
+
@@ -115,7 +119,8 @@ export default function StatusPanel(props: StatusPanelProps) { set_is_images_visible(!is_images_visible); }} style={{ textTransform: 'none' }} - startIcon={} + startIcon={} + disabled={camera_choice === 'Render Camera'} > {cameras_button} diff --git a/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx b/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx index 90fe975e0..5218b82b8 100644 --- a/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx +++ b/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx @@ -9,6 +9,8 @@ import OpenWithIcon from '@mui/icons-material/OpenWith'; import PublicOffSharpIcon from '@mui/icons-material/PublicOffSharp'; import PublicSharpIcon from '@mui/icons-material/PublicSharp'; import SyncOutlinedIcon from '@mui/icons-material/SyncOutlined'; +import ThreeDRotationIcon from '@mui/icons-material/ThreeDRotation'; +import VideoCameraBackIcon from '@mui/icons-material/VideoCameraBackOutlined'; import WebRtcWindow from '../WebRtcWindow/WebRtcWindow'; import { WebSocketContext } from '../WebSocket/WebSocket'; @@ -19,27 +21,36 @@ function CameraToggle() { const camera_choice = useSelector( (state) => state.renderingState.camera_choice, ); - const set_camera_choice = (event: SelectChangeEvent) => { - const value = event.target.value; - dispatch({ - type: 'write', - path: 'renderingState/camera_choice', - data: value, - }); + const set_camera_choice = (event: SelectChangeEvent, value: string[]) => { + if (value != null) { + dispatch({ + type: 'write', + path: 'renderingState/camera_choice', + data: value, + }); + } }; return ( - Viewport - Render View + + + Viewport + + + + Render View + ); } @@ -105,6 +116,7 @@ export default function ViewerWindow(props) { const camera_choice = useSelector( (state) => state.renderingState.camera_choice, ); + const camera_type = useSelector((state) => state.renderingState.camera_type); // listen to the viewport width const size = new THREE.Vector2(); @@ -137,21 +149,16 @@ export default function ViewerWindow(props) { labelRenderer.setSize(viewportWidth, viewportHeight); }; - // update the camera information in the python server - const sendCamera = () => { - if (websocket.readyState === WebSocket.OPEN) { - const cmd = 'write'; - const path = 'renderingState/camera'; - const data_packet = sceneTree.metadata.camera.toJSON(); - data_packet.object.timestamp = +new Date(); - const data = { - type: cmd, - path, - data: data_packet, - }; - const message = msgpack.encode(data); - websocket.send(message); - } + const render = () => { + const fps = 24; + const interval = 1000 / fps; + handleResize(); + sceneTree.metadata.camera.updateProjectionMatrix(); + sceneTree.metadata.moveCamera(); + sceneTree.metadata.camera_controls.update(interval); + requestAnimationFrame(render); + renderer.render(scene, sceneTree.metadata.camera); + labelRenderer.render(scene, sceneTree.metadata.camera); }; useEffect(() => { @@ -169,35 +176,13 @@ export default function ViewerWindow(props) { height: get_window_height(), width: get_window_width(), }); + render(); return () => { window.removeEventListener('resize', update_dimensions); }; }, []); - // keep sending the camera often - // rerun this when the websocket changes - useEffect(() => { - const fps = 24; - const interval = 1000 / fps; - const refreshIntervalId = setInterval(sendCamera, interval); - return () => { - clearInterval(refreshIntervalId); - }; - }, [websocket]); - - const render = () => { - const fps = 24; - const interval = 1000 / fps; - handleResize(); - sceneTree.metadata.camera.updateProjectionMatrix(); - sceneTree.metadata.moveCamera(); - sceneTree.metadata.camera_controls.update(interval); - requestAnimationFrame(render); - renderer.render(scene, sceneTree.metadata.camera); - labelRenderer.render(scene, sceneTree.metadata.camera); - }; - // start the three.js rendering loop // when the DOM is ready useEffect(() => { @@ -205,11 +190,6 @@ export default function ViewerWindow(props) { render(); }, []); - // updates the current camera with the field of view - useEffect(() => { - sceneTree.metadata.camera.fov = field_of_view; - }, [field_of_view]); - const render_height = useSelector( (state) => state.renderingState.render_height, ); @@ -254,8 +234,44 @@ export default function ViewerWindow(props) { const fl_new = fl * render_viewport_apsect_ratio; const fov = Math.atan(1 / fl_new) / (Math.PI / 360); sceneTree.metadata.camera.fov = fov; + } else { + sceneTree.metadata.camera.fov = 50; } + // update the camera information in the python server + const sendCamera = () => { + if (websocket.readyState === WebSocket.OPEN) { + const cmd = 'write'; + const path = 'renderingState/camera'; + const data_packet = sceneTree.metadata.camera.toJSON(); + data_packet.object.timestamp = +new Date(); + if (camera_choice === 'Render Camera') { + data_packet.object.camera_type = camera_type; + } else { + data_packet.object.camera_type = 'perspective'; + } + data_packet.object.render_aspect = render_aspect; + const data = { + type: cmd, + path, + data: data_packet, + }; + const message = msgpack.encode(data); + websocket.send(message); + } + }; + + // keep sending the camera often + // rerun this when the websocket changes + useEffect(() => { + const fps = 24; + const interval = 1000 / fps; + const refreshIntervalId = setInterval(sendCamera, interval); + return () => { + clearInterval(refreshIntervalId); + }; + }, [websocket, camera_choice, camera_type, render_aspect]); + return ( <> {/* the webrtc viewer needs to know the camera pose */} diff --git a/nerfstudio/viewer/app/src/reducer.js b/nerfstudio/viewer/app/src/reducer.js index 412bb4e7c..e5a3c9907 100644 --- a/nerfstudio/viewer/app/src/reducer.js +++ b/nerfstudio/viewer/app/src/reducer.js @@ -22,6 +22,7 @@ const initialState = { render_height: 1080, render_width: 1920, field_of_view: 50, + camera_type: 'perspective', isTraining: true, output_options: ['rgb'], // populated by the possible Graph outputs diff --git a/nerfstudio/viewer/app/src/themes/theme.ts b/nerfstudio/viewer/app/src/themes/theme.ts index 554445561..4a973209e 100644 --- a/nerfstudio/viewer/app/src/themes/theme.ts +++ b/nerfstudio/viewer/app/src/themes/theme.ts @@ -126,10 +126,13 @@ export const appTheme = createTheme({ styleOverrides: { root: { color: '#eeeeee', - backgroundColor: '#393e46', + backgroundColor: 'FFD369', '&.Mui-selected': { - color: '#FFD369', - backgroundColor: '#555555', + color: '#222831', + backgroundColor: '#FFD369', + ':hover': { + backgroundColor: '#FFD369', + }, }, '&.Mui-disabled': { color: '#999999', diff --git a/nerfstudio/viewer/server/utils.py b/nerfstudio/viewer/server/utils.py index 6d13f9378..9c3e003ec 100644 --- a/nerfstudio/viewer/server/utils.py +++ b/nerfstudio/viewer/server/utils.py @@ -77,8 +77,19 @@ def get_intrinsics_matrix_and_camera_to_world_h( image_width = aspect * image_height pp_w = image_width / 2.0 pp_h = image_height / 2.0 - focal_length = three_js_perspective_camera_focal_length(fov, image_height) - intrinsics_matrix = torch.tensor([[focal_length, 0, pp_w], [0, focal_length, pp_h], [0, 0, 1]]).float() + if (camera_object["camera_type"] == "perspective") | (camera_object["camera_type"] == "fisheye"): + focal_length = three_js_perspective_camera_focal_length(fov, image_height) + intrinsics_matrix = torch.tensor([[focal_length, 0, pp_w], [0, focal_length, pp_h], [0, 0, 1]]).float() + elif camera_object["camera_type"] == "equirectangular": + render_aspect = camera_object["render_aspect"] + if aspect < render_aspect: + intrinsics_matrix = torch.tensor( + [[pp_w, 0, pp_w], [0, image_width / render_aspect, pp_h], [0, 0, 1]] + ).float() + else: + intrinsics_matrix = torch.tensor( + [[image_height * render_aspect / 2, 0, pp_w], [0, pp_h * 2, pp_h], [0, 0, 1]] + ).float() # extrinsics camera_to_world_h = torch.tensor(get_chunks(camera_object["matrix"], size_of_chunk=4)).T.float() diff --git a/nerfstudio/viewer/server/viewer_utils.py b/nerfstudio/viewer/server/viewer_utils.py index ce70426f6..9fa47aec4 100644 --- a/nerfstudio/viewer/server/viewer_utils.py +++ b/nerfstudio/viewer/server/viewer_utils.py @@ -37,7 +37,7 @@ from aiortc import ( from cryptography.utils import CryptographyDeprecationWarning from rich.console import Console -from nerfstudio.cameras.cameras import Cameras +from nerfstudio.cameras.cameras import Cameras, CameraType from nerfstudio.cameras.rays import RayBundle from nerfstudio.configs import base_config as cfg from nerfstudio.data.datasets.base_dataset import InputDataset @@ -454,6 +454,19 @@ class ViewerState: else: self.prev_camera_matrix = camera_object["matrix"] self.camera_moving = True + + output_type = self.vis["renderingState/output_choice"].read() + if output_type is None: + output_type = OutputTypes.INIT + if self.prev_output_type != output_type: + self.camera_moving = True + + colormap_type = self.vis["renderingState/colormap_choice"].read() + if colormap_type is None: + colormap_type = ColormapTypes.INIT + if self.prev_colormap_type != colormap_type: + self.camera_moving = True + return camera_object def _apply_colormap(self, outputs: Dict[str, Any], colors: torch.Tensor = None, eps=1e-6): @@ -763,11 +776,22 @@ class ViewerState: if times is not None: times = torch.tensor([float(times)]) + camera_type_msg = camera_object["camera_type"] + if camera_type_msg == "perspective": + camera_type = CameraType.PERSPECTIVE + elif camera_type_msg == "fisheye": + camera_type = CameraType.FISHEYE + elif camera_type_msg == "equirectangular": + camera_type = CameraType.EQUIRECTANGULAR + else: + camera_type = CameraType.PERSPECTIVE + camera = Cameras( fx=intrinsics_matrix[0, 0], fy=intrinsics_matrix[1, 1], cx=intrinsics_matrix[0, 2], cy=intrinsics_matrix[1, 2], + camera_type=camera_type, camera_to_worlds=camera_to_world[None, ...], times=times, ) diff --git a/pyproject.toml b/pyproject.toml index fd57c6245..5f001e53b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "nerfstudio" -version = "0.1.13" +version = "0.1.14" description = "All-in-one repository for state-of-the-art NeRFs" readme = "README.md" license = { text="Apache 2.0"} @@ -19,7 +19,7 @@ dependencies = [ "appdirs>=1.4", "av==9.2.0", "tyro>=0.3.31", - "gdown==4.5.1", + "gdown==4.6.0", "ninja==1.10.2.3", "functorch==0.2.1", "h5py>=2.9.0", diff --git a/scripts/benchmarking/launch_train_blender.sh b/scripts/benchmarking/launch_train_blender.sh index 332ec95ef..2ddb0b008 100755 --- a/scripts/benchmarking/launch_train_blender.sh +++ b/scripts/benchmarking/launch_train_blender.sh @@ -79,9 +79,9 @@ for dataset in "${DATASETS[@]}"; do ns-train "${method_name}" "${method_opts[@]}" \ --data="data/blender/${dataset}${trans_file}" \ --experiment-name="blender_${dataset}_${tag}" \ - --trainer.relative-model-dir=nerfstudio_models/ \ - --trainer.steps-per-save=1000 \ - --trainer.max-num-iterations=16500 \ + --relative-model-dir=nerfstudio_models/ \ + --steps-per-save=1000 \ + --max-num-iterations=16500 \ --logging.local-writer.enable=False \ --logging.enable-profiler=False \ --vis "${vis}" \ diff --git a/scripts/downloads/download_data.py b/scripts/downloads/download_data.py index c444991a9..12a0a0227 100755 --- a/scripts/downloads/download_data.py +++ b/scripts/downloads/download_data.py @@ -15,6 +15,7 @@ from rich.console import Console from typing_extensions import Annotated from nerfstudio.configs.base_config import PrintableConfig +from nerfstudio.utils.scripts import run_command CONSOLE = Console(width=120) @@ -84,17 +85,39 @@ def grab_file_id(zip_url: str) -> str: return s.split("/")[0] +nerfstudio_dataset = [ + "Egypt", + "person", + "kitchen", + "plane", + "dozer", + "floating-tree", + "aspen", + "stump", + "sculpture", + "Giannini-Hall", +] nerfstudio_file_ids = { "bww_entrance": grab_file_id("https://drive.google.com/file/d/1ylkRHtfB3n3IRLf2wplpfxzPTq7nES9I/view?usp=sharing"), "campanile": grab_file_id("https://drive.google.com/file/d/13aOfGJRRH05pOOk9ikYGTwqFc2L1xskU/view?usp=sharing"), "desolation": grab_file_id("https://drive.google.com/file/d/14IzOOQm9KBJ3kPbunQbUTHPnXnmZus-f/view?usp=sharing"), - "dozer": grab_file_id("https://drive.google.com/file/d/1-OR5F_V5S4s-yzxohbwTylaXjzYLu8ZR/view?usp=sharing"), "library": grab_file_id("https://drive.google.com/file/d/1Hjbh_-BuaWETQExn2x2qGD74UwrFugHx/view?usp=sharing"), "poster": grab_file_id("https://drive.google.com/file/d/1dmjWGXlJnUxwosN6MVooCDQe970PkD-1/view?usp=sharing"), "redwoods2": grab_file_id("https://drive.google.com/file/d/1rg-4NoXT8p6vkmbWxMOY6PSG4j3rfcJ8/view?usp=sharing"), "storefront": grab_file_id("https://drive.google.com/file/d/16b792AguPZWDA_YC4igKCwXJqW0Tb21o/view?usp=sharing"), "vegetation": grab_file_id("https://drive.google.com/file/d/1wBhLQ2odycrtU39y2akVurXEAt9SsVI3/view?usp=sharing"), + "Egypt": "https://data.nerf.studio/nerfstudio/Egypt.zip", + "person": "https://data.nerf.studio/nerfstudio/person.zip", + "kitchen": "https://data.nerf.studio/nerfstudio/kitchen.zip", + "plane": "https://data.nerf.studio/nerfstudio/plane.zip", + "dozer": "https://data.nerf.studio/nerfstudio/dozer.zip", + "floating-tree": "https://data.nerf.studio/nerfstudio/floating-tree.zip", + "aspen": "https://data.nerf.studio/nerfstudio/aspen.zip", + "stump": "https://data.nerf.studio/nerfstudio/stump.zip", + "sculpture": "https://data.nerf.studio/nerfstudio/sculpture.zip", + "Giannini-Hall": "https://data.nerf.studio/nerfstudio/Giannini-Hall.zip", "all": None, + "nerfstudio-dataset": nerfstudio_dataset, } NerfstudioCaptureName = tyro.extras.literal_type_from_choices(nerfstudio_file_ids.keys()) @@ -103,14 +126,33 @@ NerfstudioCaptureName = tyro.extras.literal_type_from_choices(nerfstudio_file_id def download_capture_name(save_dir: Path, dataset_name: str, capture_name: str, capture_name_to_file_id: dict): """Download specific captures a given dataset and capture name.""" - url = f"https://drive.google.com/uc?id={capture_name_to_file_id[capture_name]}" - target_path = str(save_dir / f"{dataset_name}/{capture_name}") - os.makedirs(target_path, exist_ok=True) - download_path = Path(f"{target_path}.zip") - tmp_path = str(save_dir / ".temp") - shutil.rmtree(tmp_path, ignore_errors=True) - os.makedirs(tmp_path, exist_ok=True) - gdown.download(url, output=str(download_path)) + file_id_or_zip_url = capture_name_to_file_id[capture_name] + if file_id_or_zip_url.endswith(".zip"): + url = file_id_or_zip_url # zip url + target_path = str(save_dir / f"{dataset_name}/{capture_name}") + os.makedirs(target_path, exist_ok=True) + download_path = Path(f"{target_path}.zip") + tmp_path = str(save_dir / ".temp") + shutil.rmtree(tmp_path, ignore_errors=True) + os.makedirs(tmp_path, exist_ok=True) + try: + os.remove(download_path) + except OSError: + pass + run_command(f"wget {url} -O {download_path}", verbose=True) + else: + url = f"https://drive.google.com/uc?id={file_id_or_zip_url}" # file id + target_path = str(save_dir / f"{dataset_name}/{capture_name}") + os.makedirs(target_path, exist_ok=True) + download_path = Path(f"{target_path}.zip") + tmp_path = str(save_dir / ".temp") + shutil.rmtree(tmp_path, ignore_errors=True) + os.makedirs(tmp_path, exist_ok=True) + try: + os.remove(download_path) + except OSError: + pass + gdown.download(url, output=str(download_path)) with zipfile.ZipFile(download_path, "r") as zip_ref: zip_ref.extractall(tmp_path) inner_folders = os.listdir(tmp_path) @@ -132,7 +174,13 @@ class NerfstudioDownload(DatasetDownload): """Download the nerfstudio dataset.""" if self.capture_name == "all": for capture_name in nerfstudio_file_ids: - if capture_name != "all": + if capture_name not in ("all", "nerfstudio-dataset"): + download_capture_name(save_dir, "nerfstudio", capture_name, nerfstudio_file_ids) + return + + if self.capture_name == "nerfstudio-dataset": + for capture_name in nerfstudio_dataset: + if capture_name not in ("all", "nerfstudio-dataset"): download_capture_name(save_dir, "nerfstudio", capture_name, nerfstudio_file_ids) return diff --git a/scripts/exporter.py b/scripts/exporter.py index 658d5e191..175cee187 100644 --- a/scripts/exporter.py +++ b/scripts/exporter.py @@ -57,7 +57,7 @@ class ExportPointCloud(Exporter): bounding_box_min: Tuple[float, float, float] = (-1, -1, -1) """Minimum of the bounding box, used if use_bounding_box is True.""" bounding_box_max: Tuple[float, float, float] = (1, 1, 1) - """Minimum of the bounding box, used if use_bounding_box is True.""" + """Maximum of the bounding box, used if use_bounding_box is True.""" num_rays_per_batch: int = 32768 """Number of rays to evaluate per batch. Decrease if you run out of memory.""" std_ratio: float = 10.0 @@ -314,18 +314,20 @@ class ExportMarchingCubesMesh(Exporter): raise NotImplementedError("Marching cubes not implemented yet.") -Commands = Union[ - Annotated[ExportPointCloud, tyro.conf.subcommand(name="pointcloud")], - Annotated[ExportTSDFMesh, tyro.conf.subcommand(name="tsdf")], - Annotated[ExportPoissonMesh, tyro.conf.subcommand(name="poisson")], - Annotated[ExportMarchingCubesMesh, tyro.conf.subcommand(name="marching-cubes")], +Commands = tyro.conf.FlagConversionOff[ + Union[ + Annotated[ExportPointCloud, tyro.conf.subcommand(name="pointcloud")], + Annotated[ExportTSDFMesh, tyro.conf.subcommand(name="tsdf")], + Annotated[ExportPoissonMesh, tyro.conf.subcommand(name="poisson")], + Annotated[ExportMarchingCubesMesh, tyro.conf.subcommand(name="marching-cubes")], + ] ] def entrypoint(): """Entrypoint for use with pyproject scripts.""" tyro.extras.set_accent_color("bright_yellow") - tyro.cli(tyro.conf.FlagConversionOff[Commands]).main() + tyro.cli(Commands).main() if __name__ == "__main__": diff --git a/scripts/render.py b/scripts/render.py index c8bb4fdae..30b551228 100644 --- a/scripts/render.py +++ b/scripts/render.py @@ -5,7 +5,10 @@ render.py from __future__ import annotations import json +import os +import struct import sys +from contextlib import ExitStack from dataclasses import dataclass, field from pathlib import Path from typing import List, Optional @@ -25,7 +28,7 @@ from rich.progress import ( from typing_extensions import Literal, assert_never from nerfstudio.cameras.camera_paths import get_path_from_json, get_spiral_path -from nerfstudio.cameras.cameras import Cameras +from nerfstudio.cameras.cameras import Cameras, CameraType from nerfstudio.pipelines.base_pipeline import Pipeline from nerfstudio.utils import install_checks from nerfstudio.utils.eval_utils import eval_setup @@ -39,9 +42,12 @@ def _render_trajectory_video( cameras: Cameras, output_filename: Path, rendered_output_names: List[str], + render_width: int, + render_height: int, rendered_resolution_scaling_factor: float = 1.0, seconds: float = 5.0, output_format: Literal["images", "video"] = "video", + camera_type: CameraType = CameraType.PERSPECTIVE, ) -> None: """Helper function to create a video of the spiral trajectory. @@ -50,14 +56,17 @@ def _render_trajectory_video( cameras: Cameras to render. output_filename: Name of the output file. rendered_output_names: List of outputs to visualise. + render_width: Video width to render. + render_height: Video height to render. rendered_resolution_scaling_factor: Scaling factor to apply to the camera image resolution. seconds: Length of output video. output_format: How to save output data. + camera_type: Camera projection format type. """ - CONSOLE.print("[bold green]Creating trajectory video") - images = [] + CONSOLE.print("[bold green]Creating trajectory " + output_format) cameras.rescale_output_resolution(rendered_resolution_scaling_factor) cameras = cameras.to(pipeline.device) + fps = len(cameras) / seconds progress = Progress( TextColumn(":movie_camera: Rendering :movie_camera:"), @@ -66,37 +75,131 @@ def _render_trajectory_video( ItersPerSecColumn(suffix="fps"), TimeRemainingColumn(elapsed_when_finished=True, compact=True), ) - output_image_dir = output_filename.parent / output_filename.stem if output_format == "images": + output_image_dir = output_filename.parent / output_filename.stem output_image_dir.mkdir(parents=True, exist_ok=True) - with progress: - for camera_idx in progress.track(range(cameras.size), description=""): - camera_ray_bundle = cameras.generate_rays(camera_indices=camera_idx) - with torch.no_grad(): - outputs = pipeline.model.get_outputs_for_camera_ray_bundle(camera_ray_bundle) - render_image = [] - for rendered_output_name in rendered_output_names: - if rendered_output_name not in outputs: - CONSOLE.rule("Error", style="red") - CONSOLE.print(f"Could not find {rendered_output_name} in the model outputs", justify="center") - CONSOLE.print(f"Please set --rendered_output_name to one of: {outputs.keys()}", justify="center") - sys.exit(1) - output_image = outputs[rendered_output_name].cpu().numpy() - render_image.append(output_image) - render_image = np.concatenate(render_image, axis=1) - if output_format == "images": - media.write_image(output_image_dir / f"{camera_idx:05d}.png", render_image) - else: - images.append(render_image) - if output_format == "video": - fps = len(images) / seconds # make the folder if it doesn't exist output_filename.parent.mkdir(parents=True, exist_ok=True) - with CONSOLE.status("[yellow]Saving video", spinner="bouncingBall"): - media.write_video(output_filename, images, fps=fps) - CONSOLE.rule("[green] :tada: :tada: :tada: Success :tada: :tada: :tada:") - CONSOLE.print(f"[green]Saved video to {output_filename}", justify="center") + # NOTE: + # we could use ffmpeg_args "-movflags faststart" for progressive download, + # which would force moov atom into known position before mdat, + # but then we would have to move all of mdat to insert metadata atom + # (unless we reserve enough space to overwrite with our uuid tag, + # but we don't know how big the video file will be, so it's not certain!) + + with ExitStack() as stack: + writer = ( + stack.enter_context( + media.VideoWriter( + path=output_filename, + shape=( + int(render_height * rendered_resolution_scaling_factor), + int(render_width * rendered_resolution_scaling_factor) * len(rendered_output_names), + ), + fps=fps, + ) + ) + if output_format == "video" + else None + ) + with progress: + for camera_idx in progress.track(range(cameras.size), description=""): + camera_ray_bundle = cameras.generate_rays(camera_indices=camera_idx) + with torch.no_grad(): + outputs = pipeline.model.get_outputs_for_camera_ray_bundle(camera_ray_bundle) + render_image = [] + for rendered_output_name in rendered_output_names: + if rendered_output_name not in outputs: + CONSOLE.rule("Error", style="red") + CONSOLE.print(f"Could not find {rendered_output_name} in the model outputs", justify="center") + CONSOLE.print( + f"Please set --rendered_output_name to one of: {outputs.keys()}", justify="center" + ) + sys.exit(1) + output_image = outputs[rendered_output_name].cpu().numpy() + if output_image.shape[-1] == 1: + output_image = np.concatenate((output_image,) * 3, axis=-1) + render_image.append(output_image) + render_image = np.concatenate(render_image, axis=1) + if output_format == "images": + media.write_image(output_image_dir / f"{camera_idx:05d}.png", render_image) + if output_format == "video" and writer is not None: + writer.add_image(render_image) + + if output_format == "video": + if camera_type == CameraType.EQUIRECTANGULAR: + insert_spherical_metadata_into_file(output_filename) + + +def insert_spherical_metadata_into_file( + output_filename: Path, +) -> None: + """Inserts spherical metadata into MP4 video file in-place. + Args: + output_filename: Name of the (input and) output file. + """ + # NOTE: + # because we didn't use faststart, the moov atom will be at the end; + # to insert our metadata, we need to find (skip atoms until we get to) moov. + # we should have 0x00000020 ftyp, then 0x00000008 free, then variable mdat. + spherical_uuid = b"\xff\xcc\x82\x63\xf8\x55\x4a\x93\x88\x14\x58\x7a\x02\x52\x1f\xdd" + spherical_metadata = bytes( + """ +equirectangular +True +True +nerfstudio +""", + "utf-8", + ) + insert_size = len(spherical_metadata) + 8 + 16 + with open(output_filename, mode="r+b") as mp4file: + try: + # get file size + mp4file_size = os.stat(output_filename).st_size + + # find moov container (probably after ftyp, free, mdat) + while True: + pos = mp4file.tell() + size, tag = struct.unpack(">I4s", mp4file.read(8)) + if tag == b"moov": + break + mp4file.seek(pos + size) + # if moov isn't at end, bail + if pos + size != mp4file_size: + # TODO: to support faststart, rewrite all stco offsets + raise Exception("moov container not at end of file") + # go back and write inserted size + mp4file.seek(pos) + mp4file.write(struct.pack(">I", size + insert_size)) + # go inside moov + mp4file.seek(pos + 8) + # find trak container (probably after mvhd) + while True: + pos = mp4file.tell() + size, tag = struct.unpack(">I4s", mp4file.read(8)) + if tag == b"trak": + break + mp4file.seek(pos + size) + # go back and write inserted size + mp4file.seek(pos) + mp4file.write(struct.pack(">I", size + insert_size)) + # we need to read everything from end of trak to end of file in order to insert + # TODO: to support faststart, make more efficient (may load nearly all data) + mp4file.seek(pos + size) + rest_of_file = mp4file.read(mp4file_size - pos - size) + # go to end of trak (again) + mp4file.seek(pos + size) + # insert our uuid atom with spherical metadata + mp4file.write(struct.pack(">I4s16s", insert_size, b"uuid", spherical_uuid)) + mp4file.write(spherical_metadata) + # write rest of file + mp4file.write(rest_of_file) + finally: + mp4file.close() @dataclass @@ -138,11 +241,24 @@ class RenderTrajectory: if self.traj == "spiral": camera_start = pipeline.datamanager.eval_dataloader.get_camera(image_idx=0).flatten() # TODO(ethan): pass in the up direction of the camera + camera_type = CameraType.PERSPECTIVE + render_width = 952 + render_height = 736 camera_path = get_spiral_path(camera_start, steps=30, radius=0.1) elif self.traj == "filename": with open(self.camera_path_filename, "r", encoding="utf-8") as f: camera_path = json.load(f) seconds = camera_path["seconds"] + if "camera_type" not in camera_path: + camera_type = CameraType.PERSPECTIVE + elif camera_path["camera_type"] == "fisheye": + camera_type = CameraType.FISHEYE + elif camera_path["camera_type"] == "equirectangular": + camera_type = CameraType.EQUIRECTANGULAR + else: + camera_type = CameraType.PERSPECTIVE + render_width = camera_path["render_width"] + render_height = camera_path["render_height"] camera_path = get_path_from_json(camera_path) else: assert_never(self.traj) @@ -155,6 +271,9 @@ class RenderTrajectory: rendered_resolution_scaling_factor=1.0 / self.downscale_factor, seconds=seconds, output_format=self.output_format, + camera_type=camera_type, + render_width=render_width, + render_height=render_height, ) diff --git a/scripts/train.py b/scripts/train.py index 910dbff1c..240fdeca0 100755 --- a/scripts/train.py +++ b/scripts/train.py @@ -46,9 +46,8 @@ import yaml from rich.console import Console from nerfstudio.configs.config_utils import convert_markup_to_ansi -from nerfstudio.configs.experiment_config import ExperimentConfig from nerfstudio.configs.method_configs import AnnotatedBaseConfigUnion -from nerfstudio.engine.trainer import Trainer +from nerfstudio.engine.trainer import TrainerConfig from nerfstudio.utils import comms, profiler CONSOLE = Console(width=120) @@ -74,7 +73,7 @@ def _set_random_seed(seed) -> None: torch.manual_seed(seed) -def train_loop(local_rank: int, world_size: int, config: ExperimentConfig, global_rank: int = 0): +def train_loop(local_rank: int, world_size: int, config: TrainerConfig, global_rank: int = 0): """Main training function that sets up and runs the trainer per process Args: @@ -83,7 +82,7 @@ def train_loop(local_rank: int, world_size: int, config: ExperimentConfig, globa config: config file specifying training regimen """ _set_random_seed(config.machine.seed + global_rank) - trainer = Trainer(config, local_rank, world_size) + trainer = config.setup(local_rank=local_rank, world_size=world_size) trainer.setup() trainer.train() @@ -95,7 +94,7 @@ def _distributed_worker( num_gpus_per_machine: int, machine_rank: int, dist_url: str, - config: ExperimentConfig, + config: TrainerConfig, timeout: timedelta = DEFAULT_TIMEOUT, ) -> Any: """Spawned distributed worker that handles the initialization of process group and handles the @@ -110,7 +109,7 @@ def _distributed_worker( dist_url: URL to connect to for distributed jobs, including protocol E.g., "tcp://127.0.0.1:8686". It can be set to "auto" to automatically select a free port on localhost. - config: ExperimentConfig specifying training regimen. + config: TrainerConfig specifying training regimen. timeout: Timeout of the distributed workers. Raises: @@ -150,7 +149,7 @@ def launch( num_machines: int = 1, machine_rank: int = 0, dist_url: str = "auto", - config: Optional[ExperimentConfig] = None, + config: Optional[TrainerConfig] = None, timeout: timedelta = DEFAULT_TIMEOUT, ) -> None: """Function that spawns muliple processes to call on main_func @@ -161,7 +160,7 @@ def launch( num_machines (int, optional): total number of machines machine_rank (int, optional): rank of this machine. dist_url (str, optional): url to connect to for distributed jobs. - config (ExperimentConfig, optional): config file specifying training regimen. + config (TrainerConfig, optional): config file specifying training regimen. timeout (timedelta, optional): timeout of the distributed workers. """ assert config is not None @@ -215,7 +214,7 @@ def launch( profiler.flush_profiler(config.logging) -def main(config: ExperimentConfig) -> None: +def main(config: TrainerConfig) -> None: """Main function.""" config.set_timestamp() @@ -223,9 +222,9 @@ def main(config: ExperimentConfig) -> None: CONSOLE.log("Using --data alias for --data.pipeline.datamanager.dataparser.data") config.pipeline.datamanager.dataparser.data = config.data - if config.trainer.load_config: - CONSOLE.log(f"Loading pre-set config from: {config.trainer.load_config}") - config = yaml.load(config.trainer.load_config.read_text(), Loader=yaml.Loader) + if config.load_config: + CONSOLE.log(f"Loading pre-set config from: {config.load_config}") + config = yaml.load(config.load_config.read_text(), Loader=yaml.Loader) # print and save config config.print_to_terminal() diff --git a/tests/data/minimal_parser/train.npz b/tests/data/minimal_parser/train.npz new file mode 100644 index 000000000..9d8acbb15 Binary files /dev/null and b/tests/data/minimal_parser/train.npz differ diff --git a/tests/data/minimal_parser/val.npz b/tests/data/minimal_parser/val.npz new file mode 100644 index 000000000..9d8acbb15 Binary files /dev/null and b/tests/data/minimal_parser/val.npz differ diff --git a/tests/test_train.py b/tests/test_train.py index f7e860275..63a8099e9 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -9,18 +9,19 @@ from pathlib import Path import pytest -from nerfstudio.configs.experiment_config import ExperimentConfig from nerfstudio.configs.method_configs import method_configs from nerfstudio.data.dataparsers.blender_dataparser import BlenderDataParserConfig +from nerfstudio.data.dataparsers.minimal_dataparser import MinimalDataParserConfig +from nerfstudio.engine.trainer import TrainerConfig from scripts.train import train_loop BLACKLIST = ["base", "semantic-nerfw", "instant-ngp", "instant-ngp-bounded", "nerfacto", "phototourism"] -def set_reduced_config(config: ExperimentConfig): +def set_reduced_config(config: TrainerConfig): """Reducing the config settings to speedup test""" config.machine.num_gpus = 0 - config.trainer.max_num_iterations = 2 + config.max_num_iterations = 2 # reduce dataset factors; set dataset to test config.pipeline.datamanager.dataparser = BlenderDataParserConfig(data=Path("tests/data/lego_test")) config.pipeline.datamanager.train_num_images_to_sample_from = 1 @@ -60,5 +61,14 @@ def test_train(): train_loop(local_rank=0, world_size=0, config=config) +def test_simple_io(): + """test to check minimal data IO works correctly""" + config = method_configs["vanilla-nerf"] + config.pipeline.datamanager.dataparser = MinimalDataParserConfig(data=Path("tests/data/minimal_parser")) + config = set_reduced_config(config) + train_loop(local_rank=0, world_size=0, config=config) + + if __name__ == "__main__": test_train() + test_simple_io()