From e0f0cd05aa06bdd148a20ed62271d49ff0a6ac9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ruilong=20Li=28=E6=9D=8E=E7=91=9E=E9=BE=99=29?= Date: Fri, 22 Sep 2023 23:04:02 -0700 Subject: [PATCH 01/23] Migrate doc from RTD to gh-pages (#2460) * gh-pages workflow * update doc.yml * update * remove RTD * change action to main * test fail on warning * fail on warnings * get back .readthedocs.yml, a bit cleanup * revert SHA of index.md file --- .github/workflows/doc.yml | 33 +++++++++++++++++++++++++++++++++ docs/Makefile | 2 +- docs/make.bat | 1 + 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/doc.yml diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml new file mode 100644 index 000000000..51d07554d --- /dev/null +++ b/.github/workflows/doc.yml @@ -0,0 +1,33 @@ +name: Docs +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + contents: write +jobs: + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + with: + python-version: '3.9' + - name: Install dependencies + run: | + pip install .[docs] + - name: Sphinx build + # fail on warnings + run: | + sphinx-build docs _build -W --keep-going + - name: Deploy + uses: peaceiris/actions-gh-pages@v3 + with: + publish_branch: gh-pages + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: _build/ + force_orphan: true + # cname: docs.nerf.studio diff --git a/docs/Makefile b/docs/Makefile index d4bb2cbb9..5f8322660 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -3,7 +3,7 @@ # You can set these variables from the command line, and also # from the environment for the first two. -SPHINXOPTS ?= +SPHINXOPTS ?= -W --keep-going # build fail on warning SPHINXBUILD ?= sphinx-build SOURCEDIR = . BUILDDIR = _build diff --git a/docs/make.bat b/docs/make.bat index 32bb24529..008ca8c48 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -9,6 +9,7 @@ if "%SPHINXBUILD%" == "" ( ) set SOURCEDIR=. set BUILDDIR=_build +set SPHINXOPTS="-W --keep-going" %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( From 53037ba8a790a7aa089c8a4983c2aa210e01111c Mon Sep 17 00:00:00 2001 From: Maxim Bonnaerens Date: Sun, 24 Sep 2023 12:06:16 +0200 Subject: [PATCH 02/23] Allow custom sampler in ProposalNetworkSampler (#2464) * Allow custom sampler in ProposalNetworkSampler This allows custom (pdf)samplers in distinct repositories while still using the base ProposalNetworkSampler. * sampler should inherit from pdf sampler * docstring --- nerfstudio/model_components/ray_samplers.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nerfstudio/model_components/ray_samplers.py b/nerfstudio/model_components/ray_samplers.py index e18c680ae..7a2052b63 100644 --- a/nerfstudio/model_components/ray_samplers.py +++ b/nerfstudio/model_components/ray_samplers.py @@ -530,6 +530,7 @@ class ProposalNetworkSampler(Sampler): single_jitter: Use a same random jitter for all samples along a ray. update_sched: A function that takes the iteration number of steps between updates. initial_sampler: Sampler to use for the first iteration. Uses UniformLinDispPiecewise if not set. + pdf_sampler: PDFSampler to use after the first iteration. Uses PDFSampler if not set. """ def __init__( @@ -540,6 +541,7 @@ class ProposalNetworkSampler(Sampler): single_jitter: bool = False, update_sched: Callable = lambda x: 1, initial_sampler: Optional[Sampler] = None, + pdf_sampler: Optional[PDFSampler] = None, ) -> None: super().__init__() self.num_proposal_samples_per_ray = num_proposal_samples_per_ray @@ -554,7 +556,10 @@ class ProposalNetworkSampler(Sampler): self.initial_sampler = UniformLinDispPiecewiseSampler(single_jitter=single_jitter) else: self.initial_sampler = initial_sampler - self.pdf_sampler = PDFSampler(include_original=False, single_jitter=single_jitter) + if pdf_sampler is None: + self.pdf_sampler = PDFSampler(include_original=False, single_jitter=single_jitter) + else: + self.pdf_sampler = pdf_sampler self._anneal = 1.0 self._steps_since_update = 0 From 2864c52618fa9511639015fd390f50a38b00cd61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=C3=A1=C5=A1=20Kulh=C3=A1nek?= Date: Mon, 25 Sep 2023 17:37:08 +0200 Subject: [PATCH 03/23] Fix COLMAP dataparser masks and depthmaps (#2467) --- nerfstudio/data/dataparsers/colmap_dataparser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nerfstudio/data/dataparsers/colmap_dataparser.py b/nerfstudio/data/dataparsers/colmap_dataparser.py index c68a9cf6a..071a11e1e 100644 --- a/nerfstudio/data/dataparsers/colmap_dataparser.py +++ b/nerfstudio/data/dataparsers/colmap_dataparser.py @@ -147,11 +147,11 @@ class ColmapDataParser(DataParser): frame.update(cameras[im_data.camera_id]) if self.config.masks_path is not None: frame["mask_path"] = ( - (self.config.data / self.config.masks_path / im_data.name).with_suffix(".png").as_posix(), + (self.config.data / self.config.masks_path / im_data.name).with_suffix(".png").as_posix() ) if self.config.depths_path is not None: frame["depth_path"] = ( - (self.config.data / self.config.depths_path / im_data.name).with_suffix(".png").as_posix(), + (self.config.data / self.config.depths_path / im_data.name).with_suffix(".png").as_posix() ) frames.append(frame) if camera_model is not None: From 4cdeaa6eec186afe8fd6e1293cecd582215f1ff4 Mon Sep 17 00:00:00 2001 From: Chung Min Kim Date: Mon, 25 Sep 2023 13:14:51 -0700 Subject: [PATCH 04/23] Track all viser GUI folders to avoid folder duplicates (viewer-beta) (#2466) * Track all viser GUI folders to avoid folder duplicates * lint --------- Co-authored-by: Justin Kerr --- nerfstudio/viewer_beta/viewer.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/nerfstudio/viewer_beta/viewer.py b/nerfstudio/viewer_beta/viewer.py index 6e6e02a29..d4fbf973b 100644 --- a/nerfstudio/viewer_beta/viewer.py +++ b/nerfstudio/viewer_beta/viewer.py @@ -161,22 +161,40 @@ class Viewer: with tabs.add_tab("Export", viser.Icon.PACKAGE_EXPORT): populate_export_tab(self.viser_server, self.control_panel, config_path) - def nested_folder_install(folder_labels: List[str], element: ViewerElement): + # Keep track of the pointers to generated GUI folders, because each generated folder holds a unique ID. + viewer_gui_folders = dict() + + def nested_folder_install(folder_labels: List[str], prev_labels: List[str], element: ViewerElement): if len(folder_labels) == 0: element.install(self.viser_server) # also rewire the hook to rerender prev_cb = element.cb_hook element.cb_hook = lambda element: [prev_cb(element), self._interrupt_render(element)] else: - with self.viser_server.add_gui_folder(folder_labels[0]): - nested_folder_install(folder_labels[1:], element) + # recursively create folders + # If the folder name is "Custom Elements/a/b", then: + # in the beginning: folder_path will be + # "/".join([] + ["Custom Elements"]) --> "Custom Elements" + # later, folder_path will be + # "/".join(["Custom Elements"] + ["a"]) --> "Custom Elements/a" + # "/".join(["Custom Elements", "a"] + ["b"]) --> "Custom Elements/a/b" + # --> the element will be installed in the folder "Custom Elements/a/b" + # + # Note that the gui_folder is created only when the folder is not in viewer_gui_folders, + # and we use the folder_path as the key to check if the folder is already created. + # Otherwise, use the existing folder as context manager. + folder_path = "/".join(prev_labels + [folder_labels[0]]) + if folder_path not in viewer_gui_folders: + viewer_gui_folders[folder_path] = self.viser_server.add_gui_folder(folder_labels[0]) + with viewer_gui_folders[folder_path]: + nested_folder_install(folder_labels[1:], prev_labels + [folder_labels[0]], element) with control_tab: self.viewer_elements = [] self.viewer_elements.extend(parse_object(pipeline, ViewerElement, "Custom Elements")) for param_path, element in self.viewer_elements: folder_labels = param_path.split("/")[:-1] - nested_folder_install(folder_labels, element) + nested_folder_install(folder_labels, [], element) # scrape the trainer/pipeline for any ViewerControl objects to initialize them self.viewer_controls: List[ViewerControl] = [ From 593950efec18bdfb28297f1c9d17248f9a5e2ac0 Mon Sep 17 00:00:00 2001 From: Gina Wu <42229107+ginazhouhuiwu@users.noreply.github.com> Date: Wed, 27 Sep 2023 10:45:58 -0700 Subject: [PATCH 05/23] Update colmap installation instructions with conda (#2473) --- docs/quickstart/custom_dataset.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/quickstart/custom_dataset.md b/docs/quickstart/custom_dataset.md index c82697936..723a97c91 100644 --- a/docs/quickstart/custom_dataset.md +++ b/docs/quickstart/custom_dataset.md @@ -57,10 +57,16 @@ There are many ways to install COLMAP, unfortunately it can sometimes be a bit f ::::::{tab-set} :::::{tab-item} Linux -We recommend trying `apt`: +We recommend trying `conda`: ``` -sudo apt install colmap +conda install -c conda-forge colmap +``` + +Check that COLMAP 3.8 with CUDA is successfully installed: + +``` +colmap -h ``` If that doesn't work, you can try VKPG: From 169030199ee9e8c7e47a7f30bf542b749a4bdf06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ruilong=20Li=28=E6=9D=8E=E7=91=9E=E9=BE=99=29?= Date: Thu, 28 Sep 2023 18:14:40 -0700 Subject: [PATCH 06/23] Update doc.yml (#2479) --- .github/workflows/doc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 51d07554d..0878bdea9 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -30,4 +30,4 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: _build/ force_orphan: true - # cname: docs.nerf.studio + cname: docs.nerf.studio From 88d5b03a802c7ec6b7dc21078ec8237e942a268e Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Thu, 28 Sep 2023 18:15:35 -0700 Subject: [PATCH 07/23] Remove rtd (#2478) --- .readthedocs.yaml | 37 ------------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml deleted file mode 100644 index d66aefd16..000000000 --- a/.readthedocs.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# .readthedocs.yaml -# Read the Docs configuration file -# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details - -# Required -version: 2 - -# Set the version of Python and other tools you might need -build: - os: ubuntu-20.04 - tools: - python: '3.9' - # You can also specify other tool versions: - # nodejs: "16" - # rust: "1.55" - # golang: "1.17" - -# Build documentation in the docs/ directory with Sphinx -sphinx: - fail_on_warning: true - configuration: docs/conf.py - -# If using Sphinx, optionally build your docs in additional formats such as PDF -# formats: -# - pdf - -# Optionally declare the Python requirements required to build your docs -python: - install: - # Equivalent to 'pip install .' - - method: pip - path: . - # Equivalent to 'pip install .[docs]' - - method: pip - path: . - extra_requirements: - - docs From bbe31736f31ae3246579df9e3c256c1e81bef9af Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Fri, 29 Sep 2023 10:48:51 -0700 Subject: [PATCH 08/23] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b6f05f9be..7e1a0ad21 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ - + Documentation Status From c93b374d08b2c1e4f95032f61d55b5744e384dfd Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 29 Sep 2023 12:33:19 -0700 Subject: [PATCH 09/23] Don't deploy docs from pull requests (#2482) --- .github/workflows/doc.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 0878bdea9..7b46b3429 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -31,3 +31,4 @@ jobs: publish_dir: _build/ force_orphan: true cname: docs.nerf.studio + if: github.event_name != 'pull_request' From 7856aa006011404e0f0b4b77a6252f1ad5e8cc28 Mon Sep 17 00:00:00 2001 From: pierremerriaux-leddartech <42007976+pierremerriaux-leddartech@users.noreply.github.com> Date: Fri, 29 Sep 2023 19:49:20 -0400 Subject: [PATCH 10/23] fix:correction of PairPixelSampler in case of using masks with depth-nerfacto (#2477) fix:correction of PairPixelSampler in case of using masks --- nerfstudio/data/pixel_samplers.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/nerfstudio/data/pixel_samplers.py b/nerfstudio/data/pixel_samplers.py index 463d08294..d7e76806d 100644 --- a/nerfstudio/data/pixel_samplers.py +++ b/nerfstudio/data/pixel_samplers.py @@ -399,10 +399,11 @@ class PairPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods mask: Optional[Tensor] = None, device: Union[torch.device, str] = "cpu", ) -> Int[Tensor, "batch_size 3"]: + rays_to_sample = self.rays_to_sample if isinstance(mask, Tensor): m = erode_mask(mask.permute(0, 3, 1, 2).float(), pixel_radius=self.radius) nonzero_indices = torch.nonzero(m[:, 0], as_tuple=False).to(device) - chosen_indices = random.sample(range(len(nonzero_indices)), k=self.rays_to_sample) + chosen_indices = random.sample(range(len(nonzero_indices)), k=rays_to_sample) indices = nonzero_indices[chosen_indices] else: rays_to_sample = self.rays_to_sample @@ -418,12 +419,12 @@ class PairPixelSampler(PixelSampler): # pylint: disable=too-few-public-methods ws = torch.randint(self.radius, image_width - self.radius, s, dtype=torch.long, device=device) indices = torch.concat((ns, hs, ws), dim=1) - pair_indices = torch.hstack( - ( - torch.zeros(rays_to_sample, 1, device=device, dtype=torch.long), - torch.randint(-self.radius, self.radius, (rays_to_sample, 2), device=device, dtype=torch.long), - ) + pair_indices = torch.hstack( + ( + torch.zeros(rays_to_sample, 1, device=device, dtype=torch.long), + torch.randint(-self.radius, self.radius, (rays_to_sample, 2), device=device, dtype=torch.long), ) - pair_indices += indices - indices = torch.hstack((indices, pair_indices)).view(rays_to_sample * 2, 3) + ) + pair_indices += indices + indices = torch.hstack((indices, pair_indices)).view(rays_to_sample * 2, 3) return indices From b78d0faa1289c234f644ff296ae1f5f64ad3a05d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=C3=A1=C5=A1=20Kulh=C3=A1nek?= Date: Sun, 1 Oct 2023 22:12:48 +0200 Subject: [PATCH 11/23] ns-render - render all images (#2459) * ns-render - render all images * Fix commonprefix->commonpath * Fix raw rendering --- nerfstudio/scripts/render.py | 208 ++++++++++++++++++++++++++++++++- nerfstudio/utils/eval_utils.py | 7 +- 2 files changed, 211 insertions(+), 4 deletions(-) diff --git a/nerfstudio/scripts/render.py b/nerfstudio/scripts/render.py index 99a7cae01..ecb7961cc 100644 --- a/nerfstudio/scripts/render.py +++ b/nerfstudio/scripts/render.py @@ -23,7 +23,8 @@ import os import struct import shutil import sys -from contextlib import ExitStack +import gzip +from contextlib import ExitStack, contextmanager from dataclasses import dataclass, field from pathlib import Path from typing import Any, Dict, List, Literal, Optional, Union @@ -52,8 +53,11 @@ from nerfstudio.cameras.camera_paths import ( get_path_from_json, get_spiral_path, ) -from nerfstudio.cameras.cameras import Cameras, CameraType -from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager +from nerfstudio.cameras.cameras import Cameras, CameraType, RayBundle +from nerfstudio.data.datasets.base_dataset import Dataset +from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager, VanillaDataManagerConfig +from nerfstudio.data.utils.dataloaders import FixedIndicesEvalDataloader +from nerfstudio.engine.trainer import TrainerConfig from nerfstudio.data.scene_box import OrientedBox from nerfstudio.model_components import renderers from nerfstudio.pipelines.base_pipeline import Pipeline @@ -585,11 +589,209 @@ class SpiralRender(BaseRender): ) +@contextmanager +def _disable_datamanager_setup(cls): + """ + Disables setup_train or setup_eval for faster initialization. + """ + old_setup_train = getattr(cls, "setup_train") + old_setup_eval = getattr(cls, "setup_eval") + setattr(cls, "setup_train", lambda *args, **kwargs: None) + setattr(cls, "setup_eval", lambda *args, **kwargs: None) + yield cls + setattr(cls, "setup_train", old_setup_train) + setattr(cls, "setup_eval", old_setup_eval) + + +@dataclass +class DatasetRender(BaseRender): + """Render all images in the dataset.""" + + output_path: Path = Path("renders") + """Path to output video file.""" + data: Optional[Path] = None + """Override path to the dataset.""" + downscale_factor: Optional[float] = None + """Scaling factor to apply to the camera image resolution.""" + split: Literal["train", "val", "test", "train+test"] = "test" + """Split to render.""" + rendered_output_names: Optional[List[str]] = field(default_factory=lambda: None) + """Name of the renderer outputs to use. rgb, depth, raw-depth, gt-rgb etc. By default all outputs are rendered.""" + + def main(self): + config: TrainerConfig + + def update_config(config: TrainerConfig) -> TrainerConfig: + data_manager_config = config.pipeline.datamanager + assert isinstance(data_manager_config, VanillaDataManagerConfig) + data_manager_config.eval_image_indices = None + data_manager_config.eval_num_images_to_sample_from = -1 + data_manager_config.eval_num_times_to_repeat_images = -1 + data_manager_config.train_num_images_to_sample_from = -1 + data_manager_config.train_num_times_to_repeat_images = -1 + data_manager_config.data = self.data + if self.downscale_factor is not None: + assert hasattr(data_manager_config.dataparser, "downscale_factor") + setattr(data_manager_config.dataparser, "downscale_factor", self.downscale_factor) + return config + + config, pipeline, _, _ = eval_setup( + self.load_config, + eval_num_rays_per_chunk=self.eval_num_rays_per_chunk, + test_mode="inference", + update_config_callback=update_config, + ) + data_manager_config = config.pipeline.datamanager + assert isinstance(data_manager_config, VanillaDataManagerConfig) + + for split in self.split.split("+"): + datamanager: VanillaDataManager + dataset: Dataset + if split == "train": + with _disable_datamanager_setup(data_manager_config._target): # pylint: disable=protected-access + datamanager = data_manager_config.setup(test_mode="test", device=pipeline.device) + + dataset = datamanager.train_dataset + dataparser_outputs = getattr(dataset, "_dataparser_outputs", datamanager.train_dataparser_outputs) + else: + with _disable_datamanager_setup(data_manager_config._target): # pylint: disable=protected-access + datamanager = data_manager_config.setup(test_mode=split, device=pipeline.device) + + dataset = datamanager.eval_dataset + dataparser_outputs = getattr(dataset, "_dataparser_outputs", None) + if dataparser_outputs is None: + dataparser_outputs = datamanager.dataparser.get_dataparser_outputs(split=datamanager.test_split) + dataloader = FixedIndicesEvalDataloader( + input_dataset=dataset, + device=datamanager.device, + num_workers=datamanager.world_size * 4, + ) + images_root = Path(os.path.commonpath(dataparser_outputs.image_filenames)) + with Progress( + TextColumn(f":movie_camera: Rendering split {split} :movie_camera:"), + BarColumn(), + TaskProgressColumn( + text_format="[progress.percentage]{task.completed}/{task.total:>.0f}({task.percentage:>3.1f}%)", + show_speed=True, + ), + ItersPerSecColumn(suffix="fps"), + TimeRemainingColumn(elapsed_when_finished=False, compact=False), + TimeElapsedColumn(), + ) as progress: + for camera_idx, (ray_bundle, batch) in enumerate(progress.track(dataloader, total=len(dataset))): + ray_bundle: RayBundle + with torch.no_grad(): + outputs = pipeline.model.get_outputs_for_camera_ray_bundle(ray_bundle) + + gt_batch = batch.copy() + gt_batch["rgb"] = gt_batch.pop("image") + all_outputs = ( + list(outputs.keys()) + + [f"raw-{x}" for x in outputs.keys()] + + [f"gt-{x}" for x in gt_batch.keys()] + + [f"raw-gt-{x}" for x in gt_batch.keys()] + ) + rendered_output_names = self.rendered_output_names + if rendered_output_names is None: + rendered_output_names = ["gt-rgb"] + list(outputs.keys()) + for rendered_output_name in rendered_output_names: + if rendered_output_name not in all_outputs: + CONSOLE.rule("Error", style="red") + CONSOLE.print( + f"Could not find {rendered_output_name} in the model outputs", justify="center" + ) + CONSOLE.print( + f"Please set --rendered-output-name to one of: {all_outputs}", justify="center" + ) + sys.exit(1) + + is_raw = False + is_depth = rendered_output_name.find("depth") != -1 + image_name = f"{camera_idx:05d}" + + # Try to get the original filename + image_name = ( + dataparser_outputs.image_filenames[camera_idx].with_suffix("").relative_to(images_root) + ) + + output_path = self.output_path / split / rendered_output_name / image_name + output_path.parent.mkdir(exist_ok=True, parents=True) + + output_name = rendered_output_name + if output_name.startswith("raw-"): + output_name = output_name[4:] + is_raw = True + if output_name.startswith("gt-"): + output_name = output_name[3:] + output_image = gt_batch[output_name] + else: + output_image = outputs[output_name] + if is_depth: + # Divide by the dataparser scale factor + output_image.div_(dataparser_outputs.dataparser_scale) + else: + if output_name.startswith("gt-"): + output_name = output_name[3:] + output_image = gt_batch[output_name] + else: + output_image = outputs[output_name] + del output_name + + # Map to color spaces / numpy + if is_raw: + output_image = output_image.cpu().numpy() + elif is_depth: + output_image = ( + colormaps.apply_depth_colormap( + output_image, + accumulation=outputs["accumulation"], + near_plane=self.depth_near_plane, + far_plane=self.depth_far_plane, + colormap_options=self.colormap_options, + ) + .cpu() + .numpy() + ) + else: + output_image = ( + colormaps.apply_colormap( + image=output_image, + colormap_options=self.colormap_options, + ) + .cpu() + .numpy() + ) + + # Save to file + if is_raw: + with gzip.open(output_path.with_suffix(".npy.gz"), "wb") as f: + np.save(f, output_image) + elif self.image_format == "png": + media.write_image(output_path.with_suffix(".png"), output_image, fmt="png") + elif self.image_format == "jpeg": + media.write_image( + output_path.with_suffix(".jpg"), output_image, fmt="jpeg", quality=self.jpeg_quality + ) + else: + raise ValueError(f"Unknown image format {self.image_format}") + + table = Table( + title=None, + show_header=False, + box=box.MINIMAL, + title_style=style.Style(bold=True), + ) + for split in self.split.split("+"): + table.add_row(f"Outputs {split}", str(self.output_path / split)) + CONSOLE.print(Panel(table, title="[bold][green]:tada: Render on split {} Complete :tada:[/bold]", expand=False)) + + Commands = tyro.conf.FlagConversionOff[ Union[ Annotated[RenderCameraPath, tyro.conf.subcommand(name="camera-path")], Annotated[RenderInterpolated, tyro.conf.subcommand(name="interpolate")], Annotated[SpiralRender, tyro.conf.subcommand(name="spiral")], + Annotated[DatasetRender, tyro.conf.subcommand(name="dataset")], ] ] diff --git a/nerfstudio/utils/eval_utils.py b/nerfstudio/utils/eval_utils.py index 13948678b..7e0400536 100644 --- a/nerfstudio/utils/eval_utils.py +++ b/nerfstudio/utils/eval_utils.py @@ -20,7 +20,7 @@ from __future__ import annotations import os import sys from pathlib import Path -from typing import Literal, Optional, Tuple +from typing import Literal, Optional, Tuple, Callable import torch import yaml @@ -69,6 +69,7 @@ def eval_setup( config_path: Path, eval_num_rays_per_chunk: Optional[int] = None, test_mode: Literal["test", "val", "inference"] = "test", + update_config_callback: Optional[Callable[[TrainerConfig], TrainerConfig]] = None, ) -> Tuple[TrainerConfig, Pipeline, Path, int]: """Shared setup for loading a saved pipeline for evaluation. @@ -79,6 +80,7 @@ def eval_setup( 'val': loads train/val datasets into memory 'test': loads train/test dataset into memory 'inference': does not load any dataset into memory + update_config_callback: Callback to update the config before loading the pipeline Returns: @@ -92,6 +94,9 @@ def eval_setup( if eval_num_rays_per_chunk: config.pipeline.model.eval_num_rays_per_chunk = eval_num_rays_per_chunk + if update_config_callback is not None: + config = update_config_callback(config) + # load checkpoints from wherever they were saved # TODO: expose the ability to choose an arbitrary checkpoint config.load_dir = config.get_checkpoint_dir() From c2f5e68d548b66114a3dab0ad7707906a47126c9 Mon Sep 17 00:00:00 2001 From: Reza Kermani Date: Sun, 1 Oct 2023 23:31:02 -0400 Subject: [PATCH 12/23] fixing broken links in README (#2483) fixing broken links --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 7e1a0ad21..a853192ad 100644 --- a/README.md +++ b/README.md @@ -242,15 +242,15 @@ Using an existing dataset is great, but likely you want to use your own data! We | Data | Capture Device | Requirements | `ns-process-data` Speed | | ---------------------------------------------------------------------------------------------------- | -------------- | ----------------------------------------------------------------- | ----------------------- | -| πŸ“· [Images](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#images-and-video) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐒 | -| πŸ“Ή [Video](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#images-and-video) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐒 | -| 🌎 [360 Data](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#360_data) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐒 | -| πŸ“± [Polycam](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#polycam-capture) | IOS with LiDAR | [Polycam App](https://poly.cam/) | πŸ‡ | -| πŸ“± [KIRI Engine](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#kiri-capture) | IOS or Android | [KIRI Engine App](https://www.kiriengine.com/) | πŸ‡ | -| πŸ“± [Record3D](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#record3d-capture) | IOS with LiDAR | [Record3D app](https://record3d.app/) | πŸ‡ | -| πŸ–₯ [Metashape](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#metashape) | Any | [Metashape](https://www.agisoft.com/) | πŸ‡ | -| πŸ–₯ [RealityCapture](https://docs.nerf.studio/en/latest/quickstart/custom_dataset.html#realitycapture) | Any | [RealityCapture](https://www.capturingreality.com/realitycapture) | πŸ‡ | -| πŸ›  [Custom](https://docs.nerf.studio/en/latest/quickstart/data_conventions.html) | Any | Camera Poses | πŸ‡ | +| πŸ“· [Images](https://docs.nerf.studio/quickstart/custom_dataset.html#images-or-video) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐒 | +| πŸ“Ή [Video](https://docs.nerf.studio/quickstart/custom_dataset.html#images-or-video) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐒 | +| 🌎 [360 Data](https://docs.nerf.studio/quickstart/custom_dataset.html#data-equirectangular) | Any | [COLMAP](https://colmap.github.io/install.html) | 🐒 | +| πŸ“± [Polycam](https://docs.nerf.studio/quickstart/custom_dataset.html#polycam-capture) | IOS with LiDAR | [Polycam App](https://poly.cam/) | πŸ‡ | +| πŸ“± [KIRI Engine](https://docs.nerf.studio/quickstart/custom_dataset.html#kiri-engine-capture) | IOS or Android | [KIRI Engine App](https://www.kiriengine.com/) | πŸ‡ | +| πŸ“± [Record3D](https://docs.nerf.studio/quickstart/custom_dataset.html#record3d-capture) | IOS with LiDAR | [Record3D app](https://record3d.app/) | πŸ‡ | +| πŸ–₯ [Metashape](https://docs.nerf.studio/quickstart/custom_dataset.html#metashape) | Any | [Metashape](https://www.agisoft.com/) | πŸ‡ | +| πŸ–₯ [RealityCapture](https://docs.nerf.studio/quickstart/custom_dataset.html#realitycapture) | Any | [RealityCapture](https://www.capturingreality.com/realitycapture) | πŸ‡ | +| πŸ›  [Custom](https://docs.nerf.studio/quickstart/data_conventions.html) | Any | Camera Poses | πŸ‡ | ## 5. Advanced Options From 0d746b136347ed7b2eae6b9bd44df66f9b8edf34 Mon Sep 17 00:00:00 2001 From: Maxim Bonnaerens Date: Mon, 2 Oct 2023 23:07:34 +0200 Subject: [PATCH 13/23] Fourier Feature encodings and polyhedron encodings (#2463) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fourier Feature encodings and polyhedron encodings Rework RFFEncoding to be a subclass of the more general Fourier Feature Encodings and introduce Polyhedron encodings as introduced in mipnerf360. * b_matrix -> basis * Add typing * fighting pyright * use scale argument * continue the fight * ignore * ignore em all * add docstring and rename generate_basis to generate_polyhedron_basis * Try to please pyright with assert * Immediately allocate tensor on correct device * private functions and docstrings update * doc fix continued --------- Co-authored-by: JonΓ‘Ε‘ KulhΓ‘nek --- nerfstudio/field_components/encodings.py | 98 +++++++++++--- nerfstudio/utils/math.py | 161 ++++++++++++++++++++++- 2 files changed, 237 insertions(+), 22 deletions(-) diff --git a/nerfstudio/field_components/encodings.py b/nerfstudio/field_components/encodings.py index 437ee805f..4ba28f0c4 100644 --- a/nerfstudio/field_components/encodings.py +++ b/nerfstudio/field_components/encodings.py @@ -27,9 +27,13 @@ from jaxtyping import Float, Int, Shaped from torch import Tensor, nn from nerfstudio.field_components.base_field_component import FieldComponent -from nerfstudio.utils.math import components_from_spherical_harmonics, expected_sin +from nerfstudio.utils.external import TCNN_EXISTS, tcnn +from nerfstudio.utils.math import ( + components_from_spherical_harmonics, + expected_sin, + generate_polyhedron_basis, +) from nerfstudio.utils.printing import print_tcnn_speed_warning -from nerfstudio.utils.external import tcnn, TCNN_EXISTS class Encoding(FieldComponent): @@ -153,7 +157,7 @@ class NeRFEncoding(Encoding): Output values will be between -1 and 1 """ scaled_in_tensor = 2 * torch.pi * in_tensor # scale to [0, 2pi] - freqs = 2 ** torch.linspace(self.min_freq, self.max_freq, self.num_frequencies).to(in_tensor.device) + freqs = 2 ** torch.linspace(self.min_freq, self.max_freq, self.num_frequencies, device=in_tensor.device) scaled_inputs = scaled_in_tensor[..., None] * freqs # [..., "input_dim", "num_scales"] scaled_inputs = scaled_inputs.view(*scaled_inputs.shape[:-2], -1) # [..., "input_dim" * "num_scales"] @@ -178,34 +182,40 @@ class NeRFEncoding(Encoding): return self.pytorch_fwd(in_tensor, covs) -class RFFEncoding(Encoding): - """Random Fourier Feature encoding. Supports integrated encodings. +class FFEncoding(Encoding): + """Fourier Feature encoding. Supports integrated encodings. Args: in_dim: Input dimension of tensor - num_frequencies: Number of encoding frequencies - scale: Std of Gaussian to sample frequencies. Must be greater than zero + basis: Basis matrix from which to construct the Fourier features. + num_frequencies: Number of encoded frequencies per axis + min_freq_exp: Minimum frequency exponent + max_freq_exp: Maximum frequency exponent include_input: Append the input coordinate to the encoding """ - def __init__(self, in_dim: int, num_frequencies: int, scale: float, include_input: bool = False) -> None: + def __init__( + self, + in_dim: int, + basis: Float[Tensor, "M N"], + num_frequencies: int, + min_freq_exp: float, + max_freq_exp: float, + include_input: bool = False, + ) -> None: super().__init__(in_dim) - self.num_frequencies = num_frequencies - if not scale > 0: - raise ValueError("RFF encoding scale should be greater than zero") - self.scale = scale - if self.in_dim is None: - raise ValueError("Input dimension has not been set") - b_matrix = torch.normal(mean=0, std=self.scale, size=(self.in_dim, self.num_frequencies)) - self.register_buffer(name="b_matrix", tensor=b_matrix) + self.min_freq = min_freq_exp + self.max_freq = max_freq_exp + self.register_buffer(name="b_matrix", tensor=basis) self.include_input = include_input def get_out_dim(self) -> int: - out_dim = self.num_frequencies * 2 + if self.in_dim is None: + raise ValueError("Input dimension has not been set") + assert isinstance(self.b_matrix, Tensor) + out_dim = self.b_matrix.shape[1] * self.num_frequencies * 2 if self.include_input: - if self.in_dim is None: - raise ValueError("Input dimension has not been set") out_dim += self.in_dim return out_dim @@ -214,7 +224,7 @@ class RFFEncoding(Encoding): in_tensor: Float[Tensor, "*bs input_dim"], covs: Optional[Float[Tensor, "*bs input_dim input_dim"]] = None, ) -> Float[Tensor, "*bs output_dim"]: - """Calculates RFF encoding. If covariances are provided the encodings will be integrated as proposed + """Calculates FF encoding. If covariances are provided the encodings will be integrated as proposed in mip-NeRF. Args: @@ -226,11 +236,16 @@ class RFFEncoding(Encoding): """ scaled_in_tensor = 2 * torch.pi * in_tensor # scale to [0, 2pi] scaled_inputs = scaled_in_tensor @ self.b_matrix # [..., "num_frequencies"] + freqs = 2 ** torch.linspace(self.min_freq, self.max_freq, self.num_frequencies, device=in_tensor.device) + scaled_inputs = scaled_inputs[..., None] * freqs # [..., "input_dim", "num_scales"] + scaled_inputs = scaled_inputs.view(*scaled_inputs.shape[:-2], -1) # [..., "input_dim" * "num_scales"] if covs is None: encoded_inputs = torch.sin(torch.cat([scaled_inputs, scaled_inputs + torch.pi / 2.0], dim=-1)) else: input_var = torch.sum((covs @ self.b_matrix) * self.b_matrix, -2) + input_var = input_var[..., :, None] * freqs[None, :] ** 2 + input_var = input_var.reshape((*input_var.shape[:-2], -1)) encoded_inputs = expected_sin( torch.cat([scaled_inputs, scaled_inputs + torch.pi / 2.0], dim=-1), torch.cat(2 * [input_var], dim=-1) ) @@ -241,6 +256,49 @@ class RFFEncoding(Encoding): return encoded_inputs +class RFFEncoding(FFEncoding): + """Random Fourier Feature encoding. Supports integrated encodings. + + Args: + in_dim: Input dimension of tensor + num_frequencies: Number of encoding frequencies + scale: Std of Gaussian to sample frequencies. Must be greater than zero + include_input: Append the input coordinate to the encoding + """ + + def __init__(self, in_dim: int, num_frequencies: int, scale: float, include_input: bool = False) -> None: + if not scale > 0: + raise ValueError("RFF encoding scale should be greater than zero") + + b_matrix = torch.normal(mean=0, std=scale, size=(in_dim, num_frequencies)) + super().__init__(in_dim, b_matrix, 1, 0.0, 0.0, include_input) + + +class PolyhedronFFEncoding(FFEncoding): + """Fourier Feature encoding using polyhedron basis as proposed by mip-NeRF360. Supports integrated encodings. + + Args: + num_frequencies: Number of encoded frequencies per axis + min_freq_exp: Minimum frequency exponent + max_freq_exp: Maximum frequency exponent + basis_shape: Shape of polyhedron basis. Either "octahedron" or "icosahedron" + basis_subdivisions: Number of times to tesselate the polyhedron. + include_input: Append the input coordinate to the encoding + """ + + def __init__( + self, + num_frequencies: int, + min_freq_exp: float, + max_freq_exp: float, + basis_shape: Literal["octahedron", "icosahedron"] = "octahedron", + basis_subdivisions: int = 1, + include_input: bool = False, + ) -> None: + basis_t = generate_polyhedron_basis(basis_shape, basis_subdivisions).T + super().__init__(3, basis_t, num_frequencies, min_freq_exp, max_freq_exp, include_input) + + class HashEncoding(Encoding): """Hash encoding diff --git a/nerfstudio/utils/math.py b/nerfstudio/utils/math.py index 8567a8b28..0ba9e6a51 100644 --- a/nerfstudio/utils/math.py +++ b/nerfstudio/utils/math.py @@ -14,6 +14,8 @@ """ Math Helper Functions """ +import itertools +import math from dataclasses import dataclass from typing import Literal, Tuple @@ -195,7 +197,6 @@ def expected_sin(x_means: torch.Tensor, x_vars: torch.Tensor) -> torch.Tensor: Returns: torch.Tensor: The expected value of sin. """ - return torch.exp(-0.5 * x_vars) * torch.sin(x_means) @@ -360,4 +361,160 @@ def normalized_depth_scale_and_shift( shift[valid] = (-a_01[valid] * b_0[valid] + a_00[valid] * b_1[valid]) / det[valid] return scale, shift - return scale, shift + + +def columnwise_squared_l2_distance( + x: Float[Tensor, "*M N"], + y: Float[Tensor, "*M N"], +) -> Float[Tensor, "N N"]: + """Compute the squared Euclidean distance between all pairs of columns. + Adapted from https://github.com/google-research/multinerf/blob/5b4d4f64608ec8077222c52fdf814d40acc10bc1/internal/geopoly.py + + Args: + x: tensor of floats, with shape [M, N]. + y: tensor of floats, with shape [M, N]. + Returns: + sq_dist: tensor of floats, with shape [N, N]. + """ + # Use the fact that ||x - y||^2 == ||x||^2 + ||y||^2 - 2 x^T y. + sq_norm_x = torch.sum(x**2, 0) + sq_norm_y = torch.sum(y**2, 0) + sq_dist = sq_norm_x[:, None] + sq_norm_y[None, :] - 2 * x.T @ y + return sq_dist + + +def _compute_tesselation_weights(v: int) -> Tensor: + """Tesselate the vertices of a triangle by a factor of `v`. + Adapted from https://github.com/google-research/multinerf/blob/5b4d4f64608ec8077222c52fdf814d40acc10bc1/internal/geopoly.py + + Args: + v: int, the factor of the tesselation (v==1 is a no-op to the triangle). + + Returns: + weights: tesselated weights. + """ + if v < 1: + raise ValueError(f"v {v} must be >= 1") + int_weights = [] + for i in range(v + 1): + for j in range(v + 1 - i): + int_weights.append((i, j, v - (i + j))) + int_weights = torch.FloatTensor(int_weights) + weights = int_weights / v # Barycentric weights. + return weights + + +def _tesselate_geodesic( + vertices: Float[Tensor, "N 3"], faces: Float[Tensor, "M 3"], v: int, eps: float = 1e-4 +) -> Tensor: + """Tesselate the vertices of a geodesic polyhedron. + + Adapted from https://github.com/google-research/multinerf/blob/5b4d4f64608ec8077222c52fdf814d40acc10bc1/internal/geopoly.py + + Args: + vertices: tensor of floats, the vertex coordinates of the geodesic. + faces: tensor of ints, the indices of the vertices of base_verts that + constitute eachface of the polyhedra. + v: int, the factor of the tesselation (v==1 is a no-op). + eps: float, a small value used to determine if two vertices are the same. + + Returns: + verts: a tensor of floats, the coordinates of the tesselated vertices. + """ + tri_weights = _compute_tesselation_weights(v) + + verts = [] + for face in faces: + new_verts = torch.matmul(tri_weights, vertices[face, :]) + new_verts /= torch.sqrt(torch.sum(new_verts**2, 1, keepdim=True)) + verts.append(new_verts) + verts = torch.concatenate(verts, 0) + + sq_dist = columnwise_squared_l2_distance(verts.T, verts.T) + assignment = torch.tensor([torch.min(torch.argwhere(d <= eps)) for d in sq_dist]) + unique = torch.unique(assignment) + verts = verts[unique, :] + return verts + + +def generate_polyhedron_basis( + basis_shape: Literal["icosahedron", "octahedron"], + angular_tesselation: int, + remove_symmetries: bool = True, + eps: float = 1e-4, +) -> Tensor: + """Generates a 3D basis by tesselating a geometric polyhedron. + Basis is used to construct Fourier features for positional encoding. + See Mip-Nerf360 paper: https://arxiv.org/abs/2111.12077 + Adapted from https://github.com/google-research/multinerf/blob/5b4d4f64608ec8077222c52fdf814d40acc10bc1/internal/geopoly.py + + Args: + base_shape: string, the name of the starting polyhedron, must be either + 'icosahedron' or 'octahedron'. + angular_tesselation: int, the number of times to tesselate the polyhedron, + must be >= 1 (a value of 1 is a no-op to the polyhedron). + remove_symmetries: bool, if True then remove the symmetric basis columns, + which is usually a good idea because otherwise projections onto the basis + will have redundant negative copies of each other. + eps: float, a small number used to determine symmetries. + + Returns: + basis: a matrix with shape [3, n]. + """ + if basis_shape == "icosahedron": + a = (math.sqrt(5) + 1) / 2 + verts = torch.FloatTensor( + [ + (-1, 0, a), + (1, 0, a), + (-1, 0, -a), + (1, 0, -a), + (0, a, 1), + (0, a, -1), + (0, -a, 1), + (0, -a, -1), + (a, 1, 0), + (-a, 1, 0), + (a, -1, 0), + (-a, -1, 0), + ] + ) / math.sqrt(a + 2) + faces = torch.tensor( + [ + (0, 4, 1), + (0, 9, 4), + (9, 5, 4), + (4, 5, 8), + (4, 8, 1), + (8, 10, 1), + (8, 3, 10), + (5, 3, 8), + (5, 2, 3), + (2, 7, 3), + (7, 10, 3), + (7, 6, 10), + (7, 11, 6), + (11, 0, 6), + (0, 1, 6), + (6, 1, 10), + (9, 0, 11), + (9, 11, 2), + (9, 2, 5), + (7, 2, 11), + ] + ) + verts = _tesselate_geodesic(verts, faces, angular_tesselation) + elif basis_shape == "octahedron": + verts = torch.FloatTensor([(0, 0, -1), (0, 0, 1), (0, -1, 0), (0, 1, 0), (-1, 0, 0), (1, 0, 0)]) + corners = torch.FloatTensor(list(itertools.product([-1, 1], repeat=3))) + pairs = torch.argwhere(columnwise_squared_l2_distance(corners.T, verts.T) == 2) + faces, _ = torch.sort(torch.reshape(pairs[:, 1], [3, -1]).T, 1) + verts = _tesselate_geodesic(verts, faces, angular_tesselation) + + if remove_symmetries: + # Remove elements of `verts` that are reflections of each other. + match = columnwise_squared_l2_distance(verts.T, -verts.T) < eps + verts = verts[torch.any(torch.triu(match), 1), :] + + basis = verts.flip(-1) + return basis From afcc8434d031d85e609c5a6491998cc333d55138 Mon Sep 17 00:00:00 2001 From: Rohan Mathur Date: Tue, 3 Oct 2023 00:20:58 -0700 Subject: [PATCH 14/23] fixed link to viser to be functional link cause previous 404ed (#2486) --- docs/developer_guides/viewer/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer_guides/viewer/index.md b/docs/developer_guides/viewer/index.md index 93204a203..9df5c60d5 100644 --- a/docs/developer_guides/viewer/index.md +++ b/docs/developer_guides/viewer/index.md @@ -16,7 +16,7 @@ local_viewer We thank the authors and contributors to the following repos, which we've started, used, and modified for our use-cases. -- [Viser](https://github.com/brentyi/viser/tree/main/viser) - made by [Brent Yi](https://github.com/brentyi) +- [Viser](https://github.com/brentyi/viser/) - made by [Brent Yi](https://github.com/brentyi) - [meshcat-python](https://github.com/rdeits/meshcat-python) - made by [Robin Deits](https://github.com/rdeits) - [meshcat](https://github.com/rdeits/meshcat) - made by [Robin Deits](https://github.com/rdeits) - [ThreeJS](https://threejs.org/) From a484d255b4f71c55915afcfc52d90ec88963779f Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Wed, 4 Oct 2023 21:02:46 +0200 Subject: [PATCH 15/23] Add support of using both Comet and Viewer at the same time (#2488) --- nerfstudio/configs/experiment_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nerfstudio/configs/experiment_config.py b/nerfstudio/configs/experiment_config.py index 42cc4399e..d4b3d4de6 100644 --- a/nerfstudio/configs/experiment_config.py +++ b/nerfstudio/configs/experiment_config.py @@ -83,7 +83,7 @@ class ExperimentConfig(InstantiateConfig): def is_viewer_enabled(self) -> bool: """Checks if a viewer is enabled.""" - return ("viewer" == self.vis) | ("viewer+wandb" == self.vis) | ("viewer+tensorboard" == self.vis) + return self.vis in ("viewer", "viewer+wandb", "viewer+tensorboard", "viewer+comet") def is_viewer_beta_enabled(self) -> bool: """Checks if a viewer beta is enabled.""" From 242a1cf4fe460df4d0ed9a47b2376134eaa43a00 Mon Sep 17 00:00:00 2001 From: Ji Shi Date: Mon, 9 Oct 2023 03:59:58 +0800 Subject: [PATCH 16/23] Fix DatasetRender to override dataset path only when provided (#2494) --- nerfstudio/scripts/render.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nerfstudio/scripts/render.py b/nerfstudio/scripts/render.py index ecb7961cc..7f6102428 100644 --- a/nerfstudio/scripts/render.py +++ b/nerfstudio/scripts/render.py @@ -629,7 +629,8 @@ class DatasetRender(BaseRender): data_manager_config.eval_num_times_to_repeat_images = -1 data_manager_config.train_num_images_to_sample_from = -1 data_manager_config.train_num_times_to_repeat_images = -1 - data_manager_config.data = self.data + if self.data is not None: + data_manager_config.data = self.data if self.downscale_factor is not None: assert hasattr(data_manager_config.dataparser, "downscale_factor") setattr(data_manager_config.dataparser, "downscale_factor", self.downscale_factor) From d325593e0229ae741920416ebe1812fc0ac4af50 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Mon, 9 Oct 2023 04:35:08 +0200 Subject: [PATCH 17/23] Remove unused config param (#2495) Remove unused config. --- docs/developer_guides/pipelines/datamanagers.md | 2 -- nerfstudio/data/datamanagers/base_datamanager.py | 2 -- nerfstudio/scripts/render.py | 1 - nerfstudio/utils/eval_utils.py | 3 --- tests/data/configs/test_config1.yml | 2 -- tests/data/configs/test_config2.yml | 2 -- 6 files changed, 12 deletions(-) diff --git a/docs/developer_guides/pipelines/datamanagers.md b/docs/developer_guides/pipelines/datamanagers.md index 32c55b63e..78aec242d 100644 --- a/docs/developer_guides/pipelines/datamanagers.md +++ b/docs/developer_guides/pipelines/datamanagers.md @@ -62,8 +62,6 @@ class VanillaDataManagerConfig(InstantiateConfig): """number of rays per batch to use per eval iteration""" eval_num_images_to_sample_from: int = -1 """number of images to sample during eval iteration""" - eval_image_indices: Optional[Tuple[int, ...]] = (0,) - """specifies the image indices to use during eval; if None, uses all""" camera_optimizer: CameraOptimizerConfig = CameraOptimizerConfig() """specifies the camera pose optimizer used during training""" ``` diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index 52fd0af0f..cb63202df 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -335,8 +335,6 @@ class VanillaDataManagerConfig(DataManagerConfig): eval_num_times_to_repeat_images: int = -1 """When not evaluating on all images, number of iterations before picking new images. If -1, never pick new images.""" - eval_image_indices: Optional[Tuple[int, ...]] = (0,) - """Specifies the image indices to use during eval; if None, uses all.""" camera_optimizer: CameraOptimizerConfig = CameraOptimizerConfig() """Specifies the camera pose optimizer used during training. Helpful if poses are noisy, such as for data from Record3D.""" diff --git a/nerfstudio/scripts/render.py b/nerfstudio/scripts/render.py index 7f6102428..307bb5adb 100644 --- a/nerfstudio/scripts/render.py +++ b/nerfstudio/scripts/render.py @@ -624,7 +624,6 @@ class DatasetRender(BaseRender): def update_config(config: TrainerConfig) -> TrainerConfig: data_manager_config = config.pipeline.datamanager assert isinstance(data_manager_config, VanillaDataManagerConfig) - data_manager_config.eval_image_indices = None data_manager_config.eval_num_images_to_sample_from = -1 data_manager_config.eval_num_times_to_repeat_images = -1 data_manager_config.train_num_images_to_sample_from = -1 diff --git a/nerfstudio/utils/eval_utils.py b/nerfstudio/utils/eval_utils.py index 7e0400536..0b7306791 100644 --- a/nerfstudio/utils/eval_utils.py +++ b/nerfstudio/utils/eval_utils.py @@ -26,7 +26,6 @@ import torch import yaml from nerfstudio.configs.method_configs import all_methods -from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManagerConfig from nerfstudio.engine.trainer import TrainerConfig from nerfstudio.pipelines.base_pipeline import Pipeline from nerfstudio.utils.rich_utils import CONSOLE @@ -100,8 +99,6 @@ def eval_setup( # load checkpoints from wherever they were saved # TODO: expose the ability to choose an arbitrary checkpoint config.load_dir = config.get_checkpoint_dir() - if isinstance(config.pipeline.datamanager, VanillaDataManagerConfig): - config.pipeline.datamanager.eval_image_indices = None # setup pipeline (which includes the DataManager) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") diff --git a/tests/data/configs/test_config1.yml b/tests/data/configs/test_config1.yml index 3cf014760..67ce18cda 100644 --- a/tests/data/configs/test_config1.yml +++ b/tests/data/configs/test_config1.yml @@ -111,8 +111,6 @@ pipeline: !!python/object:nerfstudio.pipelines.base_pipeline.VanillaPipelineConf - data - varun - cheezit-scaled - eval_image_indices: !!python/tuple - - 0 eval_num_images_to_sample_from: -1 eval_num_rays_per_batch: 4096 eval_num_times_to_repeat_images: -1 diff --git a/tests/data/configs/test_config2.yml b/tests/data/configs/test_config2.yml index c1ccda379..90737789f 100644 --- a/tests/data/configs/test_config2.yml +++ b/tests/data/configs/test_config2.yml @@ -105,8 +105,6 @@ pipeline: !!python/object:nerfstudio.pipelines.base_pipeline.VanillaPipelineConf - data - varun - cheezit-scaled - eval_image_indices: !!python/tuple - - 0 eval_num_images_to_sample_from: -1 eval_num_rays_per_batch: 4096 eval_num_times_to_repeat_images: -1 From 79297c7c61749e1a8e27966f90106bab4aa53efc Mon Sep 17 00:00:00 2001 From: Ruoyu Wang Date: Mon, 9 Oct 2023 12:22:53 +0800 Subject: [PATCH 18/23] (minor) Align upsampling in TensorCPEncoding with TensorVMEncoding (#2493) * (minor) Align upsampling in TensorCPEncoding with TensorVMEncoding * Fix black --------- Co-authored-by: Matthew Tancik Co-authored-by: Matthew Tancik --- nerfstudio/field_components/encodings.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nerfstudio/field_components/encodings.py b/nerfstudio/field_components/encodings.py index 4ba28f0c4..ee2aac93c 100644 --- a/nerfstudio/field_components/encodings.py +++ b/nerfstudio/field_components/encodings.py @@ -482,9 +482,8 @@ class TensorCPEncoding(Encoding): resolution: Target resolution. """ - self.line_coef.data = F.interpolate( - self.line_coef.data, size=(resolution, 1), mode="bilinear", align_corners=True - ) + line_coef = F.interpolate(self.line_coef.data, size=(resolution, 1), mode="bilinear", align_corners=True) + self.line_coef = torch.nn.Parameter(line_coef) self.resolution = resolution From 81db1808a72b798d0e314d5bdb6abb25bd9c71c8 Mon Sep 17 00:00:00 2001 From: Jami Pekkanen Date: Mon, 9 Oct 2023 07:26:10 +0300 Subject: [PATCH 19/23] Add documentation about unknown depth pixel convention (#2490) Updated the documentation to explicate that zero value in depth images is treated as unknown depth. This behavior is defined in https://github.com/nerfstudio-project/nerfstudio/blob/a484d255b4f71c55915afcfc52d90ec88963779f/nerfstudio/model_components/losses.py#L242 Co-authored-by: Matthew Tancik --- docs/quickstart/data_conventions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quickstart/data_conventions.md b/docs/quickstart/data_conventions.md index 34324e30e..2e9e76eff 100644 --- a/docs/quickstart/data_conventions.md +++ b/docs/quickstart/data_conventions.md @@ -82,7 +82,7 @@ For a transform matrix, the first 3 columns are the +X, +Y, and +Z defining the ### Depth images -To train with depth supervision, you can also provide a `depth_file_path` for each frame in your `transforms.json` and use one of the methods that support additional depth losses (e.g., depth-nerfacto). The depths are assumed to be 16-bit or 32-bit and to be in millimeters to remain consistent with [Polyform](https://github.com/PolyCam/polyform). You can adjust this scaling factor using the `depth_unit_scale_factor` parameter in `NerfstudioDataParserConfig`. Note that by default, we resize the depth images to match the shape of the RGB images. +To train with depth supervision, you can also provide a `depth_file_path` for each frame in your `transforms.json` and use one of the methods that support additional depth losses (e.g., depth-nerfacto). The depths are assumed to be 16-bit or 32-bit and to be in millimeters to remain consistent with [Polyform](https://github.com/PolyCam/polyform). Zero-value in the depth image is treated as unknown depth. You can adjust this scaling factor using the `depth_unit_scale_factor` parameter in `NerfstudioDataParserConfig`. Note that by default, we resize the depth images to match the shape of the RGB images. ```json { From 1b85fb528b087f6ef9939addc4693f6a6e507219 Mon Sep 17 00:00:00 2001 From: Justin Kerr Date: Tue, 10 Oct 2023 13:09:09 -0700 Subject: [PATCH 20/23] patch for mkdirs on camera path json filepath (#2502) fix camera path json mkdir in viewer beta --- nerfstudio/viewer_beta/render_panel.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nerfstudio/viewer_beta/render_panel.py b/nerfstudio/viewer_beta/render_panel.py index 7342b0f38..690027f8d 100644 --- a/nerfstudio/viewer_beta/render_panel.py +++ b/nerfstudio/viewer_beta/render_panel.py @@ -625,6 +625,7 @@ def populate_render_tab( # now write the json file json_outfile = datapath / "camera_paths" / f"{render_name_text.value}.json" + json_outfile.parent.mkdir(parents=True, exist_ok=True) with open(json_outfile.absolute(), "w") as outfile: json.dump(json_data, outfile) # now show the command From e694f341a7ad49e2f6842b69cbaa95c370dd7a8a Mon Sep 17 00:00:00 2001 From: Justin Kerr Date: Tue, 10 Oct 2023 14:52:25 -0700 Subject: [PATCH 21/23] Move camera optimization out of datamanager and parallelize dataloading (#2092) --- nerfstudio/cameras/camera_optimizers.py | 99 ++++-- nerfstudio/configs/method_configs.py | 105 +++--- .../data/datamanagers/base_datamanager.py | 46 +-- .../data/datamanagers/parallel_datamanager.py | 321 ++++++++++++++++++ nerfstudio/data/pixel_samplers.py | 4 +- nerfstudio/engine/optimizers.py | 19 ++ nerfstudio/engine/trainer.py | 7 - nerfstudio/fields/nerfacto_field.py | 1 + nerfstudio/model_components/ray_generators.py | 7 +- nerfstudio/models/nerfacto.py | 18 +- nerfstudio/models/tensorf.py | 15 + nerfstudio/pipelines/base_pipeline.py | 15 +- nerfstudio/utils/tensor_dataclass.py | 8 + 13 files changed, 531 insertions(+), 134 deletions(-) create mode 100644 nerfstudio/data/datamanagers/parallel_datamanager.py diff --git a/nerfstudio/cameras/camera_optimizers.py b/nerfstudio/cameras/camera_optimizers.py index df8296ea1..7b7648884 100644 --- a/nerfstudio/cameras/camera_optimizers.py +++ b/nerfstudio/cameras/camera_optimizers.py @@ -23,19 +23,16 @@ from dataclasses import dataclass, field from typing import Literal, Optional, Type, Union import torch -import tyro from jaxtyping import Float, Int from torch import Tensor, nn from typing_extensions import assert_never from nerfstudio.cameras.lie_groups import exp_map_SE3, exp_map_SO3xR3 +from nerfstudio.cameras.rays import RayBundle from nerfstudio.configs.base_config import InstantiateConfig -from nerfstudio.engine.optimizers import AdamOptimizerConfig, OptimizerConfig -from nerfstudio.engine.schedulers import ( - ExponentialDecaySchedulerConfig, - SchedulerConfig, -) from nerfstudio.utils import poses as pose_utils +from nerfstudio.engine.optimizers import OptimizerConfig +from nerfstudio.engine.schedulers import SchedulerConfig @dataclass @@ -47,21 +44,38 @@ class CameraOptimizerConfig(InstantiateConfig): mode: Literal["off", "SO3xR3", "SE3"] = "off" """Pose optimization strategy to use. If enabled, we recommend SO3xR3.""" - position_noise_std: float = 0.0 - """Noise to add to initial positions. Useful for debugging.""" + trans_l2_penalty: float = 1e-2 + """L2 penalty on translation parameters.""" - orientation_noise_std: float = 0.0 - """Noise to add to initial orientations. Useful for debugging.""" + rot_l2_penalty: float = 1e-3 + """L2 penalty on rotation parameters.""" - optimizer: OptimizerConfig = field(default_factory=lambda: AdamOptimizerConfig(lr=6e-4, eps=1e-15)) - """ADAM parameters for camera optimization.""" + optimizer: Optional[OptimizerConfig] = field(default=None) + """Deprecated, now specified inside the optimizers dict""" - scheduler: SchedulerConfig = field(default_factory=lambda: ExponentialDecaySchedulerConfig(max_steps=10000)) - """Learning rate scheduler for camera optimizer..""" + scheduler: Optional[SchedulerConfig] = field(default=None) + """Deprecated, now specified inside the optimizers dict""" - param_group: tyro.conf.Suppress[str] = "camera_opt" - """Name of the parameter group used for pose optimization. Can be any string that doesn't conflict with other - groups.""" + def __post_init__(self): + if self.optimizer is not None: + import warnings + from nerfstudio.utils.rich_utils import CONSOLE + + CONSOLE.print( + "\noptimizer is no longer specified in the CameraOptimizerConfig, it is now defined with the rest of the param groups inside the config file under the name 'camera_opt'\n", + style="bold yellow", + ) + warnings.warn("above message coming from", FutureWarning, stacklevel=3) + + if self.scheduler is not None: + import warnings + from nerfstudio.utils.rich_utils import CONSOLE + + CONSOLE.print( + "\nscheduler is no longer specified in the CameraOptimizerConfig, it is now defined with the rest of the param groups inside the config file under the name 'camera_opt'\n", + style="bold yellow", + ) + warnings.warn("above message coming from", FutureWarning, stacklevel=3) class CameraOptimizer(nn.Module): @@ -91,16 +105,6 @@ class CameraOptimizer(nn.Module): else: assert_never(self.config.mode) - # Initialize pose noise; useful for debugging. - if config.position_noise_std != 0.0 or config.orientation_noise_std != 0.0: - assert config.position_noise_std >= 0.0 and config.orientation_noise_std >= 0.0 - std_vector = torch.tensor( - [config.position_noise_std] * 3 + [config.orientation_noise_std] * 3, device=device - ) - self.pose_noise = exp_map_SE3(torch.normal(torch.zeros((num_cameras, 6), device=device), std_vector)) - else: - self.pose_noise = None - def forward( self, indices: Int[Tensor, "camera_indices"], @@ -125,13 +129,46 @@ class CameraOptimizer(nn.Module): assert_never(self.config.mode) # Detach non-trainable indices by setting to identity transform if self.non_trainable_camera_indices is not None: - outputs[0][self.non_trainable_camera_indices] = torch.eye(4, device=self.device)[:3, :4] + if self.non_trainable_camera_indices.device != self.pose_adjustment.device: + self.non_trainable_camera_indices = self.non_trainable_camera_indices.to(self.pose_adjustment.device) + outputs[0][self.non_trainable_camera_indices] = torch.eye(4, device=self.pose_adjustment.device)[:3, :4] - # Apply initial pose noise. - if self.pose_noise is not None: - outputs.append(self.pose_noise[indices, :, :]) # Return: identity if no transforms are needed, otherwise multiply transforms together. if len(outputs) == 0: # Note that using repeat() instead of tile() here would result in unnecessary copies. return torch.eye(4, device=self.device)[None, :3, :4].tile(indices.shape[0], 1, 1) return functools.reduce(pose_utils.multiply, outputs) + + def apply_to_raybundle(self, raybundle: RayBundle) -> None: + """Apply the pose correction to the raybundle""" + if self.config.mode != "off": + correction_matrices = self(raybundle.camera_indices.squeeze()) # type: ignore + raybundle.origins = raybundle.origins + correction_matrices[:, :3, 3] + raybundle.directions = torch.bmm(correction_matrices[:, :3, :3], raybundle.directions[..., None]).squeeze() + + def get_loss_dict(self, loss_dict: dict) -> None: + """Add regularization""" + if self.config.mode != "off": + loss_dict["camera_opt_regularizer"] = ( + self.pose_adjustment[:, :3].norm(dim=-1).mean() * self.config.trans_l2_penalty + + self.pose_adjustment[:, 3:].norm(dim=-1).mean() * self.config.rot_l2_penalty + ) + + def get_correction_matrices(self): + """Get optimized pose correction matrices""" + return self(torch.arange(0, self.num_cameras).long()) + + def get_metrics_dict(self, metrics_dict: dict) -> None: + """Get camera optimizer metrics""" + if self.config.mode != "off": + metrics_dict["camera_opt_translation"] = self.pose_adjustment[:, :3].norm() + metrics_dict["camera_opt_rotation"] = self.pose_adjustment[:, 3:].norm() + + def get_param_groups(self, param_groups: dict) -> None: + """Get camera optimizer parameters""" + camera_opt_params = list(self.parameters()) + if self.config.mode != "off": + assert len(camera_opt_params) > 0 + param_groups["camera_opt"] = camera_opt_params + else: + assert len(camera_opt_params) == 0 diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index a92c042bf..74cabbafc 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -27,15 +27,23 @@ from nerfstudio.data.pixel_samplers import PairPixelSamplerConfig from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig from nerfstudio.configs.base_config import ViewerConfig from nerfstudio.configs.external_methods import get_external_methods - -from nerfstudio.data.datamanagers.random_cameras_datamanager import RandomCamerasDataManagerConfig -from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager, VanillaDataManagerConfig - +from nerfstudio.data.datamanagers.base_datamanager import ( + VanillaDataManager, + VanillaDataManagerConfig, +) +from nerfstudio.data.datamanagers.parallel_datamanager import ParallelDataManagerConfig +from nerfstudio.data.datamanagers.random_cameras_datamanager import ( + RandomCamerasDataManagerConfig, +) from nerfstudio.data.dataparsers.blender_dataparser import BlenderDataParserConfig from nerfstudio.data.dataparsers.dnerf_dataparser import DNeRFDataParserConfig -from nerfstudio.data.dataparsers.instant_ngp_dataparser import InstantNGPDataParserConfig +from nerfstudio.data.dataparsers.instant_ngp_dataparser import ( + InstantNGPDataParserConfig, +) from nerfstudio.data.dataparsers.nerfstudio_dataparser import NerfstudioDataParserConfig -from nerfstudio.data.dataparsers.phototourism_dataparser import PhototourismDataParserConfig +from nerfstudio.data.dataparsers.phototourism_dataparser import ( + PhototourismDataParserConfig, +) from nerfstudio.data.dataparsers.sdfstudio_dataparser import SDFStudioDataParserConfig from nerfstudio.data.dataparsers.sitcoms3d_dataparser import Sitcoms3DDataParserConfig from nerfstudio.data.datasets.depth_dataset import DepthDataset @@ -88,17 +96,15 @@ method_configs["nerfacto"] = TrainerConfig( max_num_iterations=30000, mixed_precision=True, pipeline=VanillaPipelineConfig( - datamanager=VanillaDataManagerConfig( + datamanager=ParallelDataManagerConfig( dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=4096, eval_num_rays_per_batch=4096, - camera_optimizer=CameraOptimizerConfig( - mode="SO3xR3", - optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2), - scheduler=ExponentialDecaySchedulerConfig(lr_final=6e-6, max_steps=200000), - ), ), - model=NerfactoModelConfig(eval_num_rays_per_chunk=1 << 15), + model=NerfactoModelConfig( + eval_num_rays_per_chunk=1 << 15, + camera_optimizer=CameraOptimizerConfig(mode="SO3xR3"), + ), ), optimizers={ "proposal_networks": { @@ -109,10 +115,15 @@ method_configs["nerfacto"] = TrainerConfig( "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), "scheduler": ExponentialDecaySchedulerConfig(lr_final=0.0001, max_steps=200000), }, + "camera_opt": { + "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=5000), + }, }, viewer=ViewerConfig(num_rays_per_chunk=1 << 15), vis="viewer", ) + method_configs["nerfacto-big"] = TrainerConfig( method_name="nerfacto", steps_per_eval_batch=500, @@ -120,14 +131,10 @@ method_configs["nerfacto-big"] = TrainerConfig( max_num_iterations=100000, mixed_precision=True, pipeline=VanillaPipelineConfig( - datamanager=VanillaDataManagerConfig( + datamanager=ParallelDataManagerConfig( dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=8192, eval_num_rays_per_batch=4096, - camera_optimizer=CameraOptimizerConfig( - mode="SO3xR3", - optimizer=RAdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-3), - ), ), model=NerfactoModelConfig( eval_num_rays_per_chunk=1 << 15, @@ -139,6 +146,7 @@ method_configs["nerfacto-big"] = TrainerConfig( max_res=4096, proposal_weights_anneal_max_num_iters=5000, log2_hashmap_size=21, + camera_optimizer=CameraOptimizerConfig(mode="SO3xR3"), ), ), optimizers={ @@ -150,10 +158,15 @@ method_configs["nerfacto-big"] = TrainerConfig( "optimizer": RAdamOptimizerConfig(lr=1e-2, eps=1e-15), "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=50000), }, + "camera_opt": { + "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=5000), + }, }, viewer=ViewerConfig(num_rays_per_chunk=1 << 15), vis="viewer", ) + method_configs["nerfacto-huge"] = TrainerConfig( method_name="nerfacto", steps_per_eval_batch=500, @@ -161,15 +174,10 @@ method_configs["nerfacto-huge"] = TrainerConfig( max_num_iterations=100000, mixed_precision=True, pipeline=VanillaPipelineConfig( - datamanager=VanillaDataManagerConfig( + datamanager=ParallelDataManagerConfig( dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=16384, eval_num_rays_per_batch=4096, - camera_optimizer=CameraOptimizerConfig( - mode="SO3xR3", - optimizer=RAdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-3), - scheduler=ExponentialDecaySchedulerConfig(lr_final=6e-5, max_steps=50000), - ), ), model=NerfactoModelConfig( eval_num_rays_per_chunk=1 << 15, @@ -185,6 +193,7 @@ method_configs["nerfacto-huge"] = TrainerConfig( max_res=8192, proposal_weights_anneal_max_num_iters=5000, log2_hashmap_size=21, + camera_optimizer=CameraOptimizerConfig(mode="SO3xR3"), ), ), optimizers={ @@ -196,6 +205,10 @@ method_configs["nerfacto-huge"] = TrainerConfig( "optimizer": RAdamOptimizerConfig(lr=1e-2, eps=1e-15), "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=50000), }, + "camera_opt": { + "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=5000), + }, }, viewer=ViewerConfig(num_rays_per_chunk=1 << 15), vis="viewer", @@ -214,11 +227,11 @@ method_configs["depth-nerfacto"] = TrainerConfig( dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=4096, eval_num_rays_per_batch=4096, - camera_optimizer=CameraOptimizerConfig( - mode="SO3xR3", optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2) - ), ), - model=DepthNerfactoModelConfig(eval_num_rays_per_chunk=1 << 15), + model=DepthNerfactoModelConfig( + eval_num_rays_per_chunk=1 << 15, + camera_optimizer=CameraOptimizerConfig(mode="SO3xR3"), + ), ), optimizers={ "proposal_networks": { @@ -229,6 +242,10 @@ method_configs["depth-nerfacto"] = TrainerConfig( "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), "scheduler": None, }, + "camera_opt": { + "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=5000), + }, }, viewer=ViewerConfig(num_rays_per_chunk=1 << 15), vis="viewer", @@ -258,7 +275,6 @@ method_configs["instant-ngp"] = TrainerConfig( vis="viewer", ) - method_configs["instant-ngp-bounded"] = TrainerConfig( method_name="instant-ngp-bounded", steps_per_eval_batch=500, @@ -286,12 +302,12 @@ method_configs["instant-ngp-bounded"] = TrainerConfig( viewer=ViewerConfig(num_rays_per_chunk=1 << 12), vis="viewer", ) - - +# +# method_configs["mipnerf"] = TrainerConfig( method_name="mipnerf", pipeline=VanillaPipelineConfig( - datamanager=VanillaDataManagerConfig(dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=1024), + datamanager=ParallelDataManagerConfig(dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=1024), model=VanillaModelConfig( _target=MipNerfModel, loss_coefficients={"rgb_loss_coarse": 0.1, "rgb_loss_fine": 1.0}, @@ -364,13 +380,14 @@ method_configs["tensorf"] = TrainerConfig( max_num_iterations=30000, mixed_precision=False, pipeline=VanillaPipelineConfig( - datamanager=VanillaDataManagerConfig( + datamanager=ParallelDataManagerConfig( dataparser=BlenderDataParserConfig(), train_num_rays_per_batch=4096, eval_num_rays_per_batch=4096, ), model=TensoRFModelConfig( regularization="tv", + camera_optimizer=CameraOptimizerConfig(mode="off"), ), ), optimizers={ @@ -382,6 +399,10 @@ method_configs["tensorf"] = TrainerConfig( "optimizer": AdamOptimizerConfig(lr=0.02), "scheduler": ExponentialDecaySchedulerConfig(lr_final=0.002, max_steps=30000), }, + "camera_opt": { + "optimizer": AdamOptimizerConfig(lr=1e-4, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-5, max_steps=5000), + }, }, viewer=ViewerConfig(num_rays_per_chunk=1 << 15), vis="viewer", @@ -420,16 +441,16 @@ method_configs["phototourism"] = TrainerConfig( dataparser=PhototourismDataParserConfig(), # NOTE: one of the only differences with nerfacto train_num_rays_per_batch=4096, eval_num_rays_per_batch=4096, - camera_optimizer=CameraOptimizerConfig( - mode="SO3xR3", optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2) - ), # Large dataset, so using prior values from VariableResDataManager. train_num_images_to_sample_from=40, train_num_times_to_repeat_images=100, eval_num_images_to_sample_from=40, eval_num_times_to_repeat_images=100, ), - model=NerfactoModelConfig(eval_num_rays_per_chunk=1 << 15), + model=NerfactoModelConfig( + eval_num_rays_per_chunk=1 << 15, + camera_optimizer=CameraOptimizerConfig(mode="SO3xR3"), + ), ), optimizers={ "proposal_networks": { @@ -440,6 +461,10 @@ method_configs["phototourism"] = TrainerConfig( "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), "scheduler": None, }, + "camera_opt": { + "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=5000), + }, }, viewer=ViewerConfig(num_rays_per_chunk=1 << 15), vis="viewer", @@ -504,9 +529,6 @@ method_configs["neus"] = TrainerConfig( dataparser=SDFStudioDataParserConfig(), train_num_rays_per_batch=1024, eval_num_rays_per_batch=1024, - camera_optimizer=CameraOptimizerConfig( - mode="off", optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2) - ), ), model=NeuSModelConfig(eval_num_rays_per_chunk=1024), ), @@ -538,9 +560,6 @@ method_configs["neus-facto"] = TrainerConfig( dataparser=SDFStudioDataParserConfig(), train_num_rays_per_batch=2048, eval_num_rays_per_batch=2048, - camera_optimizer=CameraOptimizerConfig( - mode="SO3xR3", optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2) - ), ), model=NeuSFactoModelConfig( # proposal network allows for significantly smaller sdf/color network diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index cb63202df..0b937f64c 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -113,8 +113,6 @@ class DataManagerConfig(InstantiateConfig): """Target class to instantiate.""" data: Optional[Path] = None """Source of data, may not be used by all models.""" - camera_optimizer: Optional[CameraOptimizerConfig] = None - """Specifies the camera pose optimizer used during training. Helpful if poses are noisy.""" masks_on_gpu: bool = False """Process masks on GPU for speed at the expense of memory, if True.""" images_on_gpu: bool = False @@ -335,9 +333,8 @@ class VanillaDataManagerConfig(DataManagerConfig): eval_num_times_to_repeat_images: int = -1 """When not evaluating on all images, number of iterations before picking new images. If -1, never pick new images.""" - camera_optimizer: CameraOptimizerConfig = CameraOptimizerConfig() - """Specifies the camera pose optimizer used during training. Helpful if poses are noisy, such as for data from - Record3D.""" + eval_image_indices: Optional[Tuple[int, ...]] = (0,) + """Specifies the image indices to use during eval; if None, uses all.""" collate_fn: Callable[[Any], Any] = cast(Any, staticmethod(nerfstudio_collate)) """Specifies the collate function to use for the train and eval dataloaders.""" camera_res_scale_factor: float = 1.0 @@ -346,9 +343,21 @@ class VanillaDataManagerConfig(DataManagerConfig): """ patch_size: int = 1 """Size of patch to sample from. If >1, patch-based sampling will be used.""" + camera_optimizer: Optional[CameraOptimizerConfig] = field(default=None) + """Deprecated, has been moved to the model config.""" pixel_sampler: PixelSamplerConfig = PixelSamplerConfig() """Specifies the pixel sampler used to sample pixels from images.""" + def __post_init__(self): + """Warn user of camera optimizer change.""" + if self.camera_optimizer is not None: + import warnings + + CONSOLE.print( + "\nCameraOptimizerConfig has been moved from the DataManager to the Model.\n", style="bold yellow" + ) + warnings.warn("above message coming from", FutureWarning, stacklevel=3) + TDataset = TypeVar("TDataset", bound=InputDataset, default=InputDataset) @@ -486,13 +495,7 @@ class VanillaDataManager(DataManager, Generic[TDataset]): ) self.iter_train_image_dataloader = iter(self.train_image_dataloader) self.train_pixel_sampler = self._get_pixel_sampler(self.train_dataset, self.config.train_num_rays_per_batch) - self.train_camera_optimizer = self.config.camera_optimizer.setup( - num_cameras=self.train_dataset.cameras.size, device=self.device - ) - self.train_ray_generator = RayGenerator( - self.train_dataset.cameras.to(self.device), - self.train_camera_optimizer, - ) + self.train_ray_generator = RayGenerator(self.train_dataset.cameras.to(self.device)) def setup_eval(self): """Sets up the data loader for evaluation""" @@ -510,13 +513,7 @@ class VanillaDataManager(DataManager, Generic[TDataset]): ) self.iter_eval_image_dataloader = iter(self.eval_image_dataloader) self.eval_pixel_sampler = self._get_pixel_sampler(self.eval_dataset, self.config.eval_num_rays_per_batch) - self.eval_camera_optimizer = self.config.camera_optimizer.setup( - num_cameras=self.eval_dataset.cameras.size, device=self.device - ) - self.eval_ray_generator = RayGenerator( - self.eval_dataset.cameras.to(self.device), - self.eval_camera_optimizer, - ) + self.eval_ray_generator = RayGenerator(self.eval_dataset.cameras.to(self.device)) # for loading full images self.fixed_indices_eval_dataloader = FixedIndicesEvalDataloader( input_dataset=self.eval_dataset, @@ -572,13 +569,4 @@ class VanillaDataManager(DataManager, Generic[TDataset]): Returns: A list of dictionaries containing the data manager's param groups. """ - param_groups = {} - - camera_opt_params = list(self.train_camera_optimizer.parameters()) - if self.config.camera_optimizer.mode != "off": - assert len(camera_opt_params) > 0 - param_groups[self.config.camera_optimizer.param_group] = camera_opt_params - else: - assert len(camera_opt_params) == 0 - - return param_groups + return {} diff --git a/nerfstudio/data/datamanagers/parallel_datamanager.py b/nerfstudio/data/datamanagers/parallel_datamanager.py new file mode 100644 index 000000000..8d541d79e --- /dev/null +++ b/nerfstudio/data/datamanagers/parallel_datamanager.py @@ -0,0 +1,321 @@ +# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Parallel data manager that generates training data in multiple python processes. +""" +from __future__ import annotations + +import concurrent.futures +import queue +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import ( + Dict, + Generic, + List, + Literal, + Optional, + Tuple, + Type, + Union, +) + +import torch +import torch.multiprocessing as mp +from rich.progress import track +from torch.nn import Parameter + +from nerfstudio.cameras.cameras import CameraType +from nerfstudio.cameras.rays import RayBundle +from nerfstudio.data.datamanagers.base_datamanager import ( + DataManager, + VanillaDataManagerConfig, + TDataset, + variable_res_collate, +) +from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs +from nerfstudio.data.pixel_samplers import ( + PixelSampler, + PixelSamplerConfig, + PatchPixelSamplerConfig, +) +from nerfstudio.data.utils.dataloaders import ( + CacheDataloader, + FixedIndicesEvalDataloader, + RandIndicesEvalDataloader, +) +from nerfstudio.model_components.ray_generators import RayGenerator +from nerfstudio.utils.rich_utils import CONSOLE + + +@dataclass +class ParallelDataManagerConfig(VanillaDataManagerConfig): + """Config for a `ParallelDataManager` which reads data in multiple processes""" + + _target: Type = field(default_factory=lambda: ParallelDataManager) + """Target class to instantiate.""" + num_processes: int = 1 + """Number of processes to use for train data loading. More than 1 doesn't result in that much better performance""" + queue_size: int = 2 + """Size of shared data queue containing generated ray bundles and batches. + If queue_size <= 0, the queue size is infinite.""" + max_thread_workers: Optional[int] = None + """Maximum number of threads to use in thread pool executor. If None, use ThreadPool default.""" + + +class DataProcessor(mp.Process): + """Parallel dataset batch processor. + + This class is responsible for generating ray bundles from an input dataset + in parallel python processes. + + Args: + out_queue: the output queue for storing the processed data + config: configuration object for the parallel data manager + dataparser_outputs: outputs from the dataparser + dataset: input dataset + pixel_sampler: The pixel sampler for sampling rays + """ + + def __init__( + self, + out_queue: mp.Queue, + config: ParallelDataManagerConfig, + dataparser_outputs: DataparserOutputs, + dataset: TDataset, + pixel_sampler: PixelSampler, + ): + super().__init__() + self.daemon = True + self.out_queue = out_queue + self.config = config + self.dataparser_outputs = dataparser_outputs + self.dataset = dataset + self.exclude_batch_keys_from_device = self.dataset.exclude_batch_keys_from_device + self.pixel_sampler = pixel_sampler + self.ray_generator = RayGenerator(self.dataset.cameras) + self.cache_images() + + def run(self): + """Append out queue in parallel with ray bundles and batches.""" + while True: + batch = self.pixel_sampler.sample(self.img_data) + ray_indices = batch["indices"] + ray_bundle: RayBundle = self.ray_generator(ray_indices) + # check that GPUs are available + if torch.cuda.is_available(): + ray_bundle = ray_bundle.pin_memory() + while True: + try: + self.out_queue.put_nowait((ray_bundle, batch)) + break + except queue.Full: + time.sleep(0.0001) + except Exception: + CONSOLE.print_exception() + CONSOLE.print("[bold red]Error: Error occured in parallel datamanager queue.") + + def cache_images(self): + """Caches all input images into a NxHxWx3 tensor.""" + indices = range(len(self.dataset)) + batch_list = [] + results = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=self.config.max_thread_workers) as executor: + for idx in indices: + res = executor.submit(self.dataset.__getitem__, idx) + results.append(res) + for res in track(results, description="Loading data batch", transient=False): + batch_list.append(res.result()) + self.img_data = self.config.collate_fn(batch_list) + + +class ParallelDataManager(DataManager, Generic[TDataset]): + """Data manager implementation for parallel dataloading. + + Args: + config: the DataManagerConfig used to instantiate class + """ + + def __init__( + self, + config: ParallelDataManagerConfig, + device: Union[torch.device, str] = "cpu", + test_mode: Literal["test", "val", "inference"] = "val", + world_size: int = 1, + local_rank: int = 0, + **kwargs, + ): + self.dataset_type: Type[TDataset] = kwargs.get("_dataset_type", getattr(TDataset, "__default__")) + self.config = config + self.device = device + self.world_size = world_size + self.local_rank = local_rank + self.test_mode = test_mode + self.test_split = "test" if test_mode in ["test", "inference"] else "val" + self.dataparser_config = self.config.dataparser + if self.config.data is not None: + self.config.dataparser.data = Path(self.config.data) + else: + self.config.data = self.config.dataparser.data + self.dataparser = self.dataparser_config.setup() + if test_mode == "inference": + self.dataparser.downscale_factor = 1 # Avoid opening images + self.includes_time = self.dataparser.includes_time + self.train_dataparser_outputs: DataparserOutputs = self.dataparser.get_dataparser_outputs(split="train") + self.eval_dataparser_outputs: DataparserOutputs = self.dataparser.get_dataparser_outputs(split=self.test_split) + cameras = self.train_dataparser_outputs.cameras + if len(cameras) > 1: + for i in range(1, len(cameras)): + if cameras[0].width != cameras[i].width or cameras[0].height != cameras[i].height: + CONSOLE.print("Variable resolution, using variable_res_collate") + self.config.collate_fn = variable_res_collate + break + self.train_dataset = self.create_train_dataset() + self.eval_dataset = self.create_eval_dataset() + self.exclude_batch_keys_from_device = self.train_dataset.exclude_batch_keys_from_device + # Spawn is critical for not freezing the program (PyTorch compatability issue) + # check if spawn is already set + if mp.get_start_method(allow_none=True) is None: + mp.set_start_method("spawn") + super().__init__() + + def create_train_dataset(self) -> TDataset: + """Sets up the data loaders for training.""" + return self.dataset_type( + dataparser_outputs=self.train_dataparser_outputs, + scale_factor=self.config.camera_res_scale_factor, + ) + + def create_eval_dataset(self) -> TDataset: + """Sets up the data loaders for evaluation.""" + return self.dataset_type( + dataparser_outputs=self.dataparser.get_dataparser_outputs(split=self.test_split), + scale_factor=self.config.camera_res_scale_factor, + ) + + def _get_pixel_sampler(self, dataset: TDataset, num_rays_per_batch: int) -> PixelSampler: + """Infer pixel sampler to use.""" + if self.config.patch_size > 1 and type(self.config.pixel_sampler) is PixelSamplerConfig: + return PatchPixelSamplerConfig().setup( + patch_size=self.config.patch_size, num_rays_per_batch=num_rays_per_batch + ) + is_equirectangular = (dataset.cameras.camera_type == CameraType.EQUIRECTANGULAR.value).all() + if is_equirectangular.any(): + CONSOLE.print("[bold yellow]Warning: Some cameras are equirectangular, but using default pixel sampler.") + return self.config.pixel_sampler.setup( + is_equirectangular=is_equirectangular, num_rays_per_batch=num_rays_per_batch + ) + + def setup_train(self): + """Sets up parallel python data processes for training.""" + assert self.train_dataset is not None + self.train_pix_sampler = self._get_pixel_sampler(self.train_dataset, self.config.train_num_rays_per_batch) # type: ignore + self.data_queue = mp.Manager().Queue(maxsize=self.config.queue_size) + self.data_procs = [ + DataProcessor( + out_queue=self.data_queue, # type: ignore + config=self.config, + dataparser_outputs=self.train_dataparser_outputs, + dataset=self.train_dataset, + pixel_sampler=self.train_pix_sampler, + ) + for i in range(self.config.num_processes) + ] + for proc in self.data_procs: + proc.start() + print("Started threads") + + # Prime the executor with the first batch + self.train_executor = concurrent.futures.ThreadPoolExecutor(max_workers=self.config.max_thread_workers) + self.train_batch_fut = self.train_executor.submit(self.data_queue.get) + + def setup_eval(self): + """Sets up the data loader for evaluation.""" + assert self.eval_dataset is not None + CONSOLE.print("Setting up evaluation dataset...") + self.eval_image_dataloader = CacheDataloader( + self.eval_dataset, + num_images_to_sample_from=self.config.eval_num_images_to_sample_from, + num_times_to_repeat_images=self.config.eval_num_times_to_repeat_images, + device=self.device, + num_workers=self.world_size * 4, + pin_memory=True, + collate_fn=self.config.collate_fn, + exclude_batch_keys_from_device=self.exclude_batch_keys_from_device, + ) + self.iter_eval_image_dataloader = iter(self.eval_image_dataloader) + self.eval_pixel_sampler = self._get_pixel_sampler(self.eval_dataset, self.config.eval_num_rays_per_batch) # type: ignore + self.eval_ray_generator = RayGenerator(self.eval_dataset.cameras.to(self.device)) + # for loading full images + self.fixed_indices_eval_dataloader = FixedIndicesEvalDataloader( + input_dataset=self.eval_dataset, + device=self.device, + num_workers=self.world_size * 4, + ) + self.eval_dataloader = RandIndicesEvalDataloader( + input_dataset=self.eval_dataset, + device=self.device, + num_workers=self.world_size * 4, + ) + + def next_train(self, step: int) -> Tuple[RayBundle, Dict]: + """Returns the next batch of data from the parallel training processes.""" + self.train_count += 1 + + # Fetch the next batch in an executor to parallelize the queue get() operation + # with the train step + bundle, batch = self.train_batch_fut.result() + self.train_batch_fut = self.train_executor.submit(self.data_queue.get) + ray_bundle = bundle.to(self.device) + return ray_bundle, batch + + def next_eval(self, step: int) -> Tuple[RayBundle, Dict]: + """Returns the next batch of data from the eval dataloader.""" + self.eval_count += 1 + image_batch = next(self.iter_eval_image_dataloader) + assert self.eval_pixel_sampler is not None + assert isinstance(image_batch, dict) + batch = self.eval_pixel_sampler.sample(image_batch) + ray_indices = batch["indices"] + ray_bundle = self.eval_ray_generator(ray_indices) + return ray_bundle, batch + + def next_eval_image(self, step: int) -> Tuple[int, RayBundle, Dict]: + """Retrieve the next eval image.""" + for camera_ray_bundle, batch in self.eval_dataloader: + assert camera_ray_bundle.camera_indices is not None + image_idx = int(camera_ray_bundle.camera_indices[0, 0, 0]) + return image_idx, camera_ray_bundle, batch + raise ValueError("No more eval images") + + def get_train_rays_per_batch(self) -> int: + """Returns the number of rays per batch for training.""" + return self.config.train_num_rays_per_batch + + def get_eval_rays_per_batch(self) -> int: + """Returns the number of rays per batch for evaluation.""" + return self.config.eval_num_rays_per_batch + + def get_datapath(self) -> Path: + """Returns the path to the data. This is used to determine where to save camera paths.""" + return self.config.dataparser.data + + def get_param_groups(self) -> Dict[str, List[Parameter]]: + """Get the param groups for the data manager. + Returns: + A list of dictionaries containing the data manager's param groups. + """ + return {} diff --git a/nerfstudio/data/pixel_samplers.py b/nerfstudio/data/pixel_samplers.py index d7e76806d..9234a5d42 100644 --- a/nerfstudio/data/pixel_samplers.py +++ b/nerfstudio/data/pixel_samplers.py @@ -98,7 +98,7 @@ class PixelSampler: chosen_indices = random.sample(range(len(nonzero_indices)), k=batch_size) indices = nonzero_indices[chosen_indices] else: - indices = torch.floor( + indices = ( torch.rand((batch_size, 3), device=device) * torch.tensor([num_images, image_height, image_width], device=device) ).long() @@ -172,13 +172,11 @@ class PixelSampler: collated_batch = { key: value[c, y, x] for key, value in batch.items() if key != "image_idx" and value is not None } - assert collated_batch["image"].shape[0] == num_rays_per_batch # Needed to correct the random indices to their actual camera idx locations. indices[:, 0] = batch["image_idx"][c] collated_batch["indices"] = indices # with the abs camera indices - if keep_full_image: collated_batch["full_image"] = batch["image"] diff --git a/nerfstudio/engine/optimizers.py b/nerfstudio/engine/optimizers.py index b80b77bcb..9b8d48dba 100644 --- a/nerfstudio/engine/optimizers.py +++ b/nerfstudio/engine/optimizers.py @@ -84,6 +84,25 @@ class Optimizers: self.schedulers = {} self.parameters = {} for param_group_name, params in param_groups.items(): + # For deprecation, catch the camera_opt param group and fix it nicely + if param_group_name == "camera_opt" and "camera_opt" not in config: + from nerfstudio.engine.schedulers import ExponentialDecaySchedulerConfig + from nerfstudio.utils.rich_utils import CONSOLE + + CONSOLE.print( + "\nThe 'camera_opt' param group should be assigned an optimizer in the config. Assigning default optimizers for now. This will be removed in a future release.\n", + style="bold yellow", + ) + + config["camera_opt"] = { + "optimizer": AdamOptimizerConfig(lr=1e-3, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=1e-4, max_steps=30000), + } + # Print some nice warning messages if the user forgot to specify an optimizer + if param_group_name not in config: + raise RuntimeError( + f"""Optimizer config for '{param_group_name}' not found in config file. Make sure you specify an optimizer for each parameter group. Provided configs were: {config.keys()}""" + ) lr_init = config[param_group_name]["optimizer"].lr self.optimizers[param_group_name] = config[param_group_name]["optimizer"].setup(params=params) self.parameters[param_group_name] = params diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index b4ccba9ee..fe4aa85ce 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -219,13 +219,6 @@ class Trainer: """ optimizer_config = self.config.optimizers.copy() param_groups = self.pipeline.get_param_groups() - camera_optimizer_config = self.config.pipeline.datamanager.camera_optimizer - if camera_optimizer_config is not None and camera_optimizer_config.mode != "off": - assert camera_optimizer_config.param_group not in optimizer_config - optimizer_config[camera_optimizer_config.param_group] = { - "optimizer": camera_optimizer_config.optimizer, - "scheduler": camera_optimizer_config.scheduler, - } return Optimizers(optimizer_config, param_groups) def train(self) -> None: diff --git a/nerfstudio/fields/nerfacto_field.py b/nerfstudio/fields/nerfacto_field.py index 34c6346fc..f215a0045 100644 --- a/nerfstudio/fields/nerfacto_field.py +++ b/nerfstudio/fields/nerfacto_field.py @@ -116,6 +116,7 @@ class NerfactoField(Field): self.use_pred_normals = use_pred_normals self.pass_semantic_gradients = pass_semantic_gradients self.base_res = base_res + self.step = 0 self.direction_encoding = SHEncoding( levels=4, diff --git a/nerfstudio/model_components/ray_generators.py b/nerfstudio/model_components/ray_generators.py index 49754548d..bad32258d 100644 --- a/nerfstudio/model_components/ray_generators.py +++ b/nerfstudio/model_components/ray_generators.py @@ -18,7 +18,6 @@ Ray generator. from jaxtyping import Int from torch import Tensor, nn -from nerfstudio.cameras.camera_optimizers import CameraOptimizer from nerfstudio.cameras.cameras import Cameras from nerfstudio.cameras.rays import RayBundle @@ -34,10 +33,9 @@ class RayGenerator(nn.Module): image_coords: Tensor - def __init__(self, cameras: Cameras, pose_optimizer: CameraOptimizer) -> None: + def __init__(self, cameras: Cameras) -> None: super().__init__() self.cameras = cameras - self.pose_optimizer = pose_optimizer self.register_buffer("image_coords", cameras.get_image_coords(), persistent=False) def forward(self, ray_indices: Int[Tensor, "num_rays 3"]) -> RayBundle: @@ -51,11 +49,8 @@ class RayGenerator(nn.Module): x = ray_indices[:, 2] # col indices coords = self.image_coords[y, x] - camera_opt_to_camera = self.pose_optimizer(c) - ray_bundle = self.cameras.generate_rays( camera_indices=c.unsqueeze(-1), coords=coords, - camera_opt_to_camera=camera_opt_to_camera, ) return ray_bundle diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py index 1122d5c20..df8eed9b6 100644 --- a/nerfstudio/models/nerfacto.py +++ b/nerfstudio/models/nerfacto.py @@ -28,6 +28,7 @@ from torchmetrics.functional import structural_similarity_index_measure from torchmetrics.image import PeakSignalNoiseRatio from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity +from nerfstudio.cameras.camera_optimizers import CameraOptimizer, CameraOptimizerConfig from nerfstudio.cameras.rays import RayBundle, RaySamples from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes, TrainingCallbackLocation from nerfstudio.field_components.field_heads import FieldHeadNames @@ -126,6 +127,8 @@ class NerfactoModelConfig(ModelConfig): """Which implementation to use for the model.""" appearance_embed_dim: int = 32 """Dimension of the appearance embedding.""" + camera_optimizer: CameraOptimizerConfig = CameraOptimizerConfig(mode="SO3xR3") + """Config of the camera optimizer to use""" class NerfactoModel(Model): @@ -165,6 +168,9 @@ class NerfactoModel(Model): implementation=self.config.implementation, ) + self.camera_optimizer: CameraOptimizer = self.config.camera_optimizer.setup( + num_cameras=self.num_train_data, device="cpu" + ) self.density_fns = [] num_prop_nets = self.config.num_proposal_iterations # Build the proposal network(s) @@ -229,7 +235,7 @@ class NerfactoModel(Model): # losses self.rgb_loss = MSELoss() - + self.step = 0 # metrics self.psnr = PeakSignalNoiseRatio(data_range=1.0) self.ssim = structural_similarity_index_measure @@ -240,6 +246,7 @@ class NerfactoModel(Model): param_groups = {} param_groups["proposal_networks"] = list(self.proposal_networks.parameters()) param_groups["fields"] = list(self.field.parameters()) + self.camera_optimizer.get_param_groups(param_groups=param_groups) return param_groups def get_training_callbacks( @@ -252,6 +259,7 @@ class NerfactoModel(Model): def set_anneal(step): # https://arxiv.org/pdf/2111.12077.pdf eq. 18 + self.step = step train_frac = np.clip(step / N, 0, 1) self.step = step @@ -278,6 +286,9 @@ class NerfactoModel(Model): return callbacks def get_outputs(self, ray_bundle: RayBundle): + # apply the camera optimizer pose tweaks + if self.training: + self.camera_optimizer.apply_to_raybundle(ray_bundle) ray_samples: RaySamples ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns) field_outputs = self.field.forward(ray_samples, compute_normals=self.config.predict_normals) @@ -324,7 +335,6 @@ class NerfactoModel(Model): for i in range(self.config.num_proposal_iterations): outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i]) - return outputs def get_metrics_dict(self, outputs, batch): @@ -336,6 +346,8 @@ class NerfactoModel(Model): if self.training: metrics_dict["distortion"] = distortion_loss(outputs["weights_list"], outputs["ray_samples_list"]) + + self.camera_optimizer.get_metrics_dict(metrics_dict) return metrics_dict def get_loss_dict(self, outputs, batch, metrics_dict=None): @@ -364,6 +376,8 @@ class NerfactoModel(Model): loss_dict["pred_normal_loss"] = self.config.pred_normal_loss_mult * torch.mean( outputs["rendered_pred_normal_loss"] ) + # Add loss from camera optimizer + self.camera_optimizer.get_loss_dict(loss_dict) return loss_dict def get_image_metrics_and_images( diff --git a/nerfstudio/models/tensorf.py b/nerfstudio/models/tensorf.py index a0380d42e..9cc9c7ccf 100644 --- a/nerfstudio/models/tensorf.py +++ b/nerfstudio/models/tensorf.py @@ -53,6 +53,7 @@ from nerfstudio.model_components.renderers import ( from nerfstudio.model_components.scene_colliders import AABBBoxCollider from nerfstudio.models.base_model import Model, ModelConfig from nerfstudio.utils import colormaps, colors, misc +from nerfstudio.cameras.camera_optimizers import CameraOptimizer, CameraOptimizerConfig @dataclass @@ -89,6 +90,8 @@ class TensoRFModelConfig(ModelConfig): tensorf_encoding: Literal["triplane", "vm", "cp"] = "vm" regularization: Literal["none", "l1", "tv"] = "l1" """Regularization method used in tensorf paper""" + camera_optimizer: CameraOptimizerConfig = CameraOptimizerConfig(mode="SO3xR3") + """Config of the camera optimizer to use""" background_color: Literal["random", "last_sample", "black", "white"] = "white" """Whether to randomize the background color.""" @@ -256,6 +259,11 @@ class TensoRFModel(Model): if self.config.tensorf_encoding == "cp" and self.config.regularization == "tv": raise RuntimeError("TV reg not supported for CP decomposition") + # (optional) camera optimizer + self.camera_optimizer: CameraOptimizer = self.config.camera_optimizer.setup( + num_cameras=self.num_train_data, device="cpu" + ) + def get_param_groups(self) -> Dict[str, List[Parameter]]: param_groups = {} @@ -267,11 +275,14 @@ class TensoRFModel(Model): param_groups["encodings"] = list(self.field.color_encoding.parameters()) + list( self.field.density_encoding.parameters() ) + self.camera_optimizer.get_param_groups(param_groups=param_groups) return param_groups def get_outputs(self, ray_bundle: RayBundle): # uniform sampling + if self.training: + self.camera_optimizer.apply_to_raybundle(ray_bundle) ray_samples_uniform = self.sampler_uniform(ray_bundle) dens = self.field.get_density(ray_samples_uniform) weights = ray_samples_uniform.get_weights(dens) @@ -334,6 +345,8 @@ class TensoRFModel(Model): else: raise ValueError(f"Regularization {self.config.regularization} not supported") + self.camera_optimizer.get_loss_dict(loss_dict) + loss_dict = misc.scale_dict(loss_dict, self.config.loss_coefficients) return loss_dict @@ -367,5 +380,7 @@ class TensoRFModel(Model): "ssim": float(ssim.item()), "lpips": float(lpips.item()), } + self.camera_optimizer.get_metrics_dict(metrics_dict) + images_dict = {"img": combined_rgb, "accumulation": acc, "depth": depth} return metrics_dict, images_dict diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py index 5e202b06c..10aca0e70 100644 --- a/nerfstudio/pipelines/base_pipeline.py +++ b/nerfstudio/pipelines/base_pipeline.py @@ -45,6 +45,7 @@ from nerfstudio.data.datamanagers.base_datamanager import ( DataManagerConfig, VanillaDataManager, ) +from nerfstudio.data.datamanagers.parallel_datamanager import ParallelDataManager from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes from nerfstudio.models.base_model import Model, ModelConfig from nerfstudio.utils import profiler @@ -298,18 +299,6 @@ class VanillaPipeline(Pipeline): ray_bundle, batch = self.datamanager.next_train(step) model_outputs = self._model(ray_bundle) # train distributed data parallel model if world_size > 1 metrics_dict = self.model.get_metrics_dict(model_outputs, batch) - - if self.config.datamanager.camera_optimizer is not None: - camera_opt_param_group = self.config.datamanager.camera_optimizer.param_group - if camera_opt_param_group in self.datamanager.get_param_groups(): - # Report the camera optimization metrics - metrics_dict["camera_opt_translation"] = ( - self.datamanager.get_param_groups()[camera_opt_param_group][0].data[:, :3].norm() - ) - metrics_dict["camera_opt_rotation"] = ( - self.datamanager.get_param_groups()[camera_opt_param_group][0].data[:, 3:].norm() - ) - loss_dict = self.model.get_loss_dict(model_outputs, batch, metrics_dict) return model_outputs, loss_dict, metrics_dict @@ -372,7 +361,7 @@ class VanillaPipeline(Pipeline): """ self.eval() metrics_dict_list = [] - assert isinstance(self.datamanager, VanillaDataManager) + assert isinstance(self.datamanager, (VanillaDataManager, ParallelDataManager)) num_images = len(self.datamanager.fixed_indices_eval_dataloader) with Progress( TextColumn("[progress.description]{task.description}"), diff --git a/nerfstudio/utils/tensor_dataclass.py b/nerfstudio/utils/tensor_dataclass.py index 0ebe7b5ce..a2b8d1dad 100644 --- a/nerfstudio/utils/tensor_dataclass.py +++ b/nerfstudio/utils/tensor_dataclass.py @@ -261,6 +261,14 @@ class TensorDataclass: """ return self._apply_fn_to_fields(lambda x: x.to(device)) + def pin_memory(self: TensorDataclassT) -> TensorDataclassT: + """Pins the tensor dataclass memory + + Returns: + TensorDataclass: A new TensorDataclass with the same data but pinned. + """ + return self._apply_fn_to_fields(lambda x: x.pin_memory()) + def _apply_fn_to_fields( self: TensorDataclassT, fn: Callable, From dba20fcb26d9256c4267b578f39b3614a6ffe669 Mon Sep 17 00:00:00 2001 From: Justin Kerr Date: Tue, 10 Oct 2023 17:18:16 -0700 Subject: [PATCH 22/23] minor patch for exporting pcs and meshes with parallel datamanager (#2503) * minor patch for exporting pcs and meshes with parallel datamanager * format --- nerfstudio/data/datamanagers/parallel_datamanager.py | 11 +++++++++-- nerfstudio/scripts/exporter.py | 5 +++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/nerfstudio/data/datamanagers/parallel_datamanager.py b/nerfstudio/data/datamanagers/parallel_datamanager.py index 8d541d79e..7a2e5ac55 100644 --- a/nerfstudio/data/datamanagers/parallel_datamanager.py +++ b/nerfstudio/data/datamanagers/parallel_datamanager.py @@ -222,7 +222,7 @@ class ParallelDataManager(DataManager, Generic[TDataset]): def setup_train(self): """Sets up parallel python data processes for training.""" assert self.train_dataset is not None - self.train_pix_sampler = self._get_pixel_sampler(self.train_dataset, self.config.train_num_rays_per_batch) # type: ignore + self.train_pixel_sampler = self._get_pixel_sampler(self.train_dataset, self.config.train_num_rays_per_batch) # type: ignore self.data_queue = mp.Manager().Queue(maxsize=self.config.queue_size) self.data_procs = [ DataProcessor( @@ -230,7 +230,7 @@ class ParallelDataManager(DataManager, Generic[TDataset]): config=self.config, dataparser_outputs=self.train_dataparser_outputs, dataset=self.train_dataset, - pixel_sampler=self.train_pix_sampler, + pixel_sampler=self.train_pixel_sampler, ) for i in range(self.config.num_processes) ] @@ -319,3 +319,10 @@ class ParallelDataManager(DataManager, Generic[TDataset]): A list of dictionaries containing the data manager's param groups. """ return {} + + def __del__(self): + """Clean up the parallel data processes.""" + if hasattr(self, "data_procs"): + for proc in self.data_procs: + proc.terminate() + proc.join() diff --git a/nerfstudio/scripts/exporter.py b/nerfstudio/scripts/exporter.py index 2e4fc68ca..b104597ab 100644 --- a/nerfstudio/scripts/exporter.py +++ b/nerfstudio/scripts/exporter.py @@ -34,6 +34,7 @@ from typing_extensions import Annotated, Literal from nerfstudio.cameras.rays import RayBundle from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager +from nerfstudio.data.datamanagers.parallel_datamanager import ParallelDataManager from nerfstudio.data.scene_box import OrientedBox from nerfstudio.exporter import texture_utils, tsdf_utils from nerfstudio.exporter.exporter_utils import ( @@ -137,7 +138,7 @@ class ExportPointCloud(Exporter): validate_pipeline(self.normal_method, self.normal_output_name, pipeline) # Increase the batchsize to speed up the evaluation. - assert isinstance(pipeline.datamanager, VanillaDataManager) + assert isinstance(pipeline.datamanager, (VanillaDataManager, ParallelDataManager)) assert pipeline.datamanager.train_pixel_sampler is not None pipeline.datamanager.train_pixel_sampler.num_rays_per_batch = self.num_rays_per_batch @@ -307,7 +308,7 @@ class ExportPoissonMesh(Exporter): validate_pipeline(self.normal_method, self.normal_output_name, pipeline) # Increase the batchsize to speed up the evaluation. - assert isinstance(pipeline.datamanager, VanillaDataManager) + assert isinstance(pipeline.datamanager, (VanillaDataManager, ParallelDataManager)) assert pipeline.datamanager.train_pixel_sampler is not None pipeline.datamanager.train_pixel_sampler.num_rays_per_batch = self.num_rays_per_batch From 8e2a0651d0dc9bb9d6de5d4a934d0a195ad5032d Mon Sep 17 00:00:00 2001 From: Justin Kerr Date: Tue, 10 Oct 2023 17:49:52 -0700 Subject: [PATCH 23/23] Fix jitter viewer (#2505) --- nerfstudio/viewer_beta/control_panel.py | 6 +----- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/nerfstudio/viewer_beta/control_panel.py b/nerfstudio/viewer_beta/control_panel.py index 07f556dd4..716181694 100644 --- a/nerfstudio/viewer_beta/control_panel.py +++ b/nerfstudio/viewer_beta/control_panel.py @@ -269,11 +269,7 @@ class ControlPanel: Args: step: the train step to set the model to """ - with self.viser_server.atomic(), self.stat_folder: - # TODO change to a .value call instead of remove() and add, this makes it jittery - with self.viser_server.atomic(): - self.markdown.remove() - self.markdown = self.viser_server.add_gui_markdown(f"Step: {step}") + self.markdown.content = f"Step: {step}" def update_output_options(self, new_options: List[str]): """ diff --git a/pyproject.toml b/pyproject.toml index c041858ba..fc2b0be03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ dependencies = [ "torchvision>=0.14.1", "torchmetrics[image]>=1.0.1", "typing_extensions>=4.4.0", - "viser==0.1.3", + "viser==0.1.6", "nuscenes-devkit>=1.1.1", "wandb>=0.13.3", "xatlas",