diff --git a/docs/nerfology/methods/bionerf.md b/docs/nerfology/methods/bionerf.md index 5ab344522..fccd754d2 100644 --- a/docs/nerfology/methods/bionerf.md +++ b/docs/nerfology/methods/bionerf.md @@ -2,7 +2,7 @@

Biologically Plausible Neural Radiance Fields for View Synthesis

-```{button-link} https://arxiv.org/pdf/2402.07310 +```{button-link} https://leandropassosjr.github.io/BioNeRF/ :color: primary :outline: Paper Website @@ -19,7 +19,7 @@ Official Code :outline: Nerfstudio add-on code ``` -![examples](https://leandropassosjr.github.io/BioNeRF/assets/gifs.gif)
+![examples](https://leandropassosjr.github.io/BioNeRF/static/images/gifs.gif)
**Biologically Plausible Neural Radiance Fields** @@ -43,7 +43,7 @@ ns-train bionerf --help ## Pipeline -![pipeline](https://leandropassosjr.github.io/BioNeRF/assets/BioNeRF.png)
+![pipeline](https://leandropassosjr.github.io/BioNeRF/static/images/BioNeRF.png)
Here is an overview pipeline for BioNeRF, we will walk through each component in this guide. @@ -69,7 +69,7 @@ This step is responsible for adding contextual information to BioNeRF. Two new e | SSIM | 0.927 | 0.957 | 0.965 | 0.874 | 0.978 | 0.977 | 0.963 | 0.980 | 0.953 | | LPIPS | 0.047 | 0.018 | 0.017 | 0.068 | 0.018 | 0.011 | 0.016 | 0.010 | 0.026 | - + Ground-truth (top) and synthetic view (bottom) images generated by BioNeRF regarding four Realistic Blender dataset’s scenes. @@ -81,6 +81,6 @@ Ground-truth (top) and synthetic view (bottom) images generated by BioNeRF regar | SSIM | 0.837 | 0.873 | 0.914 | 0.882 | 0.796 | 0.714 | 0.911 | 0.911 | 0.861 | | LPIPS | 0.093 | 0.055 | 0.025 | 0.070 | 0.103 | 0.122 | 0.029 | 0.044 | 0.068 | - + Ground-truth (top) and synthetic view (bottom) images generated by BioNeRF regarding four LLFF dataset’s scenes. diff --git a/nerfstudio/data/datamanagers/full_images_datamanager.py b/nerfstudio/data/datamanagers/full_images_datamanager.py index 8ed8c6170..a4e67f3ea 100644 --- a/nerfstudio/data/datamanagers/full_images_datamanager.py +++ b/nerfstudio/data/datamanagers/full_images_datamanager.py @@ -339,6 +339,10 @@ def _undistort_image( ) -> Tuple[np.ndarray, np.ndarray, Optional[torch.Tensor]]: mask = None if camera.camera_type.item() == CameraType.PERSPECTIVE.value: + assert distortion_params[3] == 0, ( + "We doesn't support the 4th Brown parameter for image undistortion, " + "Only k1, k2, k3, p1, p2 can be non-zero." + ) distortion_params = np.array( [ distortion_params[0], diff --git a/nerfstudio/data/pixel_samplers.py b/nerfstudio/data/pixel_samplers.py index 93de0a996..ad11ee409 100644 --- a/nerfstudio/data/pixel_samplers.py +++ b/nerfstudio/data/pixel_samplers.py @@ -396,6 +396,10 @@ class PatchPixelSampler(PixelSampler): config: PatchPixelSamplerConfig + def __init__(self, config: PatchPixelSamplerConfig, **kwargs) -> None: + super().__init__(config, **kwargs) + self.config.patch_size = self.kwargs.get("patch_size", self.config.patch_size) + def set_num_rays_per_batch(self, num_rays_per_batch: int): """Set the number of rays to sample per batch. Overridden to deal with patch-based sampling. diff --git a/nerfstudio/process_data/realitycapture_utils.py b/nerfstudio/process_data/realitycapture_utils.py index c18afbd44..7cf2696ae 100644 --- a/nerfstudio/process_data/realitycapture_utils.py +++ b/nerfstudio/process_data/realitycapture_utils.py @@ -16,8 +16,9 @@ import csv import json +import shutil from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Optional import numpy as np from PIL import Image @@ -29,10 +30,12 @@ from nerfstudio.utils.rich_utils import CONSOLE def realitycapture_to_json( image_filename_map: Dict[str, Path], csv_filename: Path, + ply_filename: Optional[Path], output_dir: Path, verbose: bool = False, ) -> List[str]: """Convert RealityCapture data into a nerfstudio dataset. + See https://dev.epicgames.com/community/learning/knowledge-base/vzwB/capturing-reality-realitycapture-xmp-camera-math Args: image_filenames: List of paths to the original images. @@ -73,18 +76,19 @@ def realitycapture_to_json( frame["h"] = int(height) frame["w"] = int(width) frame["file_path"] = image_filename_map[basename].as_posix() - frame["fl_x"] = float(cameras["f"][i]) * max(width, height) / 36 - frame["fl_y"] = float(cameras["f"][i]) * max(width, height) / 36 - # TODO: Unclear how to get the principal point from RealityCapture, here a guess... - frame["cx"] = float(cameras["px"][i]) / 36.0 + width / 2.0 - frame["cy"] = float(cameras["py"][i]) / 36.0 + height / 2.0 - # TODO: Not sure if RealityCapture uses this distortion model - frame["k1"] = cameras["k1"][i] - frame["k2"] = cameras["k2"][i] - frame["k3"] = cameras["k3"][i] - frame["k4"] = cameras["k4"][i] - frame["p1"] = cameras["t1"][i] - frame["p2"] = cameras["t2"][i] + # reality capture uses the 35mm equivalent focal length + # See https://en.wikipedia.org/wiki/35_mm_equivalent_focal_length + scale = max(width, height) + frame["fl_x"] = float(cameras["f"][i]) * scale / 36.0 + frame["fl_y"] = float(cameras["f"][i]) * scale / 36.0 + frame["cx"] = float(cameras["px"][i]) * scale + width / 2.0 + frame["cy"] = float(cameras["py"][i]) * scale + height / 2.0 + frame["k1"] = float(cameras["k1"][i]) + frame["k2"] = float(cameras["k2"][i]) + frame["k3"] = float(cameras["k3"][i]) + frame["k4"] = float(cameras["k4"][i]) + frame["p1"] = float(cameras["t1"][i]) + frame["p2"] = float(cameras["t2"][i]) # Transform matrix to nerfstudio format. Please refer to the documentation for coordinate system conventions. rot = _get_rotation_matrix(-float(cameras["heading"][i]), float(cameras["pitch"][i]), float(cameras["roll"][i])) @@ -97,6 +101,10 @@ def realitycapture_to_json( frames.append(frame) data["frames"] = frames + if ply_filename is not None: + shutil.copy(ply_filename, output_dir / "sparse_pc.ply") + data["ply_file_path"] = "sparse_pc.ply" + with open(output_dir / "transforms.json", "w", encoding="utf-8") as f: json.dump(data, f, indent=4) diff --git a/nerfstudio/scripts/process_data.py b/nerfstudio/scripts/process_data.py index 643587298..b2c2fa13f 100644 --- a/nerfstudio/scripts/process_data.py +++ b/nerfstudio/scripts/process_data.py @@ -336,6 +336,9 @@ class ProcessRealityCapture(BaseConverterToNerfstudioDataset, _NoDefaultProcessR 2. Converts RealityCapture poses into the nerfstudio format. """ + ply: Optional[Path] = None + """Path to the RealityCapture exported ply file""" + num_downscales: int = 3 """Number of times to downscale the images. Downscales by 2 each time. For example a value of 3 will downscale the images by 2x, 4x, and 8x.""" @@ -353,6 +356,12 @@ class ProcessRealityCapture(BaseConverterToNerfstudioDataset, _NoDefaultProcessR if self.eval_data is not None: raise ValueError("Cannot use eval_data since cameras were already aligned with RealityCapture.") + if self.ply is not None: + if self.ply.suffix != ".ply": + raise ValueError(f"PLY file {self.ply} must have a .ply extension") + if not self.ply.exists(): + raise ValueError(f"PLY file {self.ply} doesn't exist") + self.output_dir.mkdir(parents=True, exist_ok=True) image_dir = self.output_dir / "images" image_dir.mkdir(parents=True, exist_ok=True) @@ -390,7 +399,9 @@ class ProcessRealityCapture(BaseConverterToNerfstudioDataset, _NoDefaultProcessR realitycapture_utils.realitycapture_to_json( image_filename_map=image_filename_map, csv_filename=self.csv, + ply_filename=self.ply, output_dir=self.output_dir, + verbose=self.verbose, ) )