From 3096f445b8440aaafdc625ef1ec8aed088084ca8 Mon Sep 17 00:00:00 2001 From: Justin Kerr Date: Mon, 6 May 2024 14:40:05 -0700 Subject: [PATCH 1/3] Add a pointcloud export from polycam to process-data (#3124) * add a pointcloud export from polycam mesh to process-data * bugfix --- nerfstudio/process_data/polycam_utils.py | 30 +++++++++++++++++++++++- nerfstudio/scripts/process_data.py | 1 + 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/nerfstudio/process_data/polycam_utils.py b/nerfstudio/process_data/polycam_utils.py index 0b1ba7a62..a84de5a31 100644 --- a/nerfstudio/process_data/polycam_utils.py +++ b/nerfstudio/process_data/polycam_utils.py @@ -17,7 +17,9 @@ import json import sys from pathlib import Path -from typing import List, Tuple +from typing import List, Optional, Tuple + +import numpy as np from nerfstudio.process_data import process_data_utils from nerfstudio.process_data.process_data_utils import CAMERA_MODELS @@ -28,6 +30,7 @@ from nerfstudio.utils.rich_utils import CONSOLE def polycam_to_json( image_filenames: List[Path], depth_filenames: List[Path], + glb_filename: Optional[Path], cameras_dir: Path, output_dir: Path, min_blur_score: float = 0.0, @@ -80,6 +83,31 @@ def polycam_to_json( frames.append(frame) data["frames"] = frames + if glb_filename is not None: + # If the .glb is populated, use it to save a pointcloud for splatfacto init + import open3d as o3d + + mesh = o3d.io.read_triangle_mesh(str(glb_filename), enable_post_processing=True) + textures = np.asarray(mesh.textures[0]) # 2D images of color + vert_points = np.asarray(mesh.vertices) # 3D positions of verts + tri_ids = np.asarray(mesh.triangles) # indices of the vertices + points = vert_points[tri_ids.flatten()] # get the 3D positions of the vertices + uvs = np.asarray(mesh.triangle_uvs) # get the uv coords of the vertices + # convert uv coord to texture integer index + tex_ids = (uvs[:, 1] * textures.shape[0]).astype(int), (uvs[:, 0] * textures.shape[1]).astype(int) + colors = textures[tex_ids[0], tex_ids[1]] + pointcloud = o3d.geometry.PointCloud(o3d.utility.Vector3dVector(points)) + pointcloud.colors = o3d.utility.Vector3dVector(colors.astype(np.float64) / 255.0) + pointcloud = pointcloud.remove_duplicated_points() + # align the pointcloud to the coord system of cameras, which is provided inside the mesh_info.json file + mesh_info_json = json.load(open(glb_filename.parent / "mesh_info.json")) + transform = np.array(mesh_info_json["alignmentTransform"]).reshape(4, 4).T + pointcloud = pointcloud.transform(np.linalg.inv(transform)) + # shift the axes coordinates to match the nerfstudio ones (same as the cameras' coord system) + pointcloud.points = o3d.utility.Vector3dVector(np.array(pointcloud.points)[:, [2, 0, 1]]) + o3d.io.write_point_cloud(str(output_dir / "point_cloud.ply"), pointcloud) + data["ply_file_path"] = "point_cloud.ply" + with open(output_dir / "transforms.json", "w", encoding="utf-8") as f: json.dump(data, f, indent=4) diff --git a/nerfstudio/scripts/process_data.py b/nerfstudio/scripts/process_data.py index 0ab2bc8f0..643587298 100644 --- a/nerfstudio/scripts/process_data.py +++ b/nerfstudio/scripts/process_data.py @@ -200,6 +200,7 @@ class ProcessPolycam(BaseConverterToNerfstudioDataset): polycam_utils.polycam_to_json( image_filenames=polycam_image_filenames, depth_filenames=polycam_depth_filenames, + glb_filename=self.data / "raw.glb" if (self.data / "raw.glb").exists() else None, cameras_dir=polycam_cameras_dir, output_dir=self.output_dir, min_blur_score=self.min_blur_score, From 9fe8e6c27c65c438163e8117bb874f9ed5a242e3 Mon Sep 17 00:00:00 2001 From: Leandro Passos Date: Tue, 7 May 2024 02:11:07 -0300 Subject: [PATCH 2/3] BioNeRF (#3125) * bionerf markdown file * method description in markdown file * inserting bionerf in index file * bionerf description * fixing image paths * including results on readme * deleting last line from bionerfs page --- docs/index.md | 1 + docs/nerfology/methods/bionerf.md | 86 ++++++++++++++++++++++++++ docs/nerfology/methods/index.md | 1 + nerfstudio/configs/external_methods.py | 13 ++++ 4 files changed, 101 insertions(+) create mode 100644 docs/nerfology/methods/bionerf.md diff --git a/docs/index.md b/docs/index.md index 4e992418c..67aa3b9e5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -147,6 +147,7 @@ This documentation is organized into 3 parts: ### Third-party Methods +- [BioNeRF](nerfology/methods/bionerf.md): Biologically Plausible Neural Radiance Fields for View Synthesis - [Instruct-NeRF2NeRF](nerfology/methods/in2n.md): Editing 3D Scenes with Instructions - [Instruct-GS2GS](nerfology/methods/igs2gs.md): Editing 3DGS Scenes with Instructions - [SIGNeRF](nerfology/methods/signerf.md): Controlled Generative Editing of NeRF Scenes diff --git a/docs/nerfology/methods/bionerf.md b/docs/nerfology/methods/bionerf.md new file mode 100644 index 000000000..c5d3c5295 --- /dev/null +++ b/docs/nerfology/methods/bionerf.md @@ -0,0 +1,86 @@ +# BioNeRF + +

Biologically Plausible Neural Radiance Fields for View Synthesis

+ +```{button-link} https://arxiv.org/pdf/2402.07310 +:color: primary +:outline: +Paper Website +``` + +```{button-link} https://github.com/Leandropassosjr/BioNeRF +:color: primary +:outline: +Official Code +``` + +```{button-link} https://github.com/Leandropassosjr/ns_bionerf +:color: primary +:outline: +Nerfstudio add-on code +``` +![examples](https://leandropassosjr.github.io/BioNeRF/assets/gifs.gif)
+ +**Biologically Plausible Neural Radiance Fields** + +## Installation + +Install nerfstudio dependencies. + +## Running BioNeRF + +Once the nerfstudio dependencies are installed, run: + +```bash +ns-train bionerf --help +``` + +## Method + +### Overview + +[BioNeRF](https://arxiv.org/pdf/2402.07310.pdf) (Biologically Plausible Neural Radiance Fields) extends [NeRF](http://www.matthewtancik.com/nerf) by implementing a cognitive-inspired mechanism that fuses inputs from multiple sources into a memory-like structure, thus improving the storing capacity and extracting more intrinsic and correlated information. BioNeRF also mimics a behavior observed in pyramidal cells concerning contextual information, in which the memory is provided as the context and combined with the inputs of two subsequent blocks of dense layers, one responsible for producing the volumetric densities and the other the colors used to render the novel view. + +## Pipeline + +![pipeline](https://leandropassosjr.github.io/BioNeRF/assets/BioNeRF.png)
+ +Here is an overview pipeline for BioNeRF, we will walk through each component in this guide. + +### Positional Feature Extraction +The first step consists of feeding two neural models simultaneously, namely $M_{\Delta}$ and $M_c$, with the camera positional information. The output of these models encodes the positional information from the input image. Although the input is the same, the neural models do not share weights and follow a different flow in the next steps. + +### Cognitive Filtering +This step performs a series of operations, called \emph{filters}, that work on the embeddings coming from the previous step. There are four filters this step derives: density, color, memory, and modulation. + +### Memory Updating +Updating the memory requires the implementation of a mechanism capable of obliterating trivial information, which is performed using the memory filter (Step 3.1 in Figure~\ref{f.bionerf}). Fist, one needs to compute a signal modulation **$\mu$**, for further introducing new experiences in the memory **$\Psi$** through the modulating variable **$\mu$** using a $\textit{tanh}$ function (Step 3.2 in the figure). + +### Contextual Inference +This step is responsible for adding contextual information to BioNeRF. Two new embeddings are generate, i.e., **${h}^{\prime}_\Delta$** and **${h}^{\prime}_c$** based on density and color filters, respectively (Step 4 in the figure), which further feed two neural models, i.e., $M^\prime_\Delta$ and $M^{\prime}$. Subsequently, $M^{\prime}_\Delta$ outputs the volume density, while color information is predicted by $M^{\prime}_c$, further used to compute the final predicted pixel information and the loss function. + +## Results + +**Blender** (synthetic) + +| | drums | materials | ficus | ship | mic | chair | lego | hotdog | AVG | +| ----- | ----- | --------- | ----- | ---- | --- | ----- | ---- | ------ | --- | +| PSNR | 25.66 | 29.74 | 29.56 | 29.57 | 33.38 | 34.63 | 31.82 | 37.23 | 31.45 | +| SSIM | 0.927 | 0.957 | 0.965 | 0.874 | 0.978 | 0.977 | 0.963 | 0.980 | 0.953 | +| LPIPS | 0.047 | 0.018 | 0.017 | 0.068 | 0.018 | 0.011 | 0.016 | 0.010 | 0.026 | + + + +Ground-truth (top) and synthetic view (bottom) images generated by BioNeRF regarding four Realistic Blender dataset’s scenes. + +**LLFF** (real) + +| | Fern | Flower | Fortress | Horns | Leaves | Orchids | Room | T-Rex | AVG | +| ---- | ----- | ------ | -------- | ----- | ----- | ------- | ----- | ----- | --- | +| PSNR | 25.17 | 27.89 | 32.34 | 27.99 | 22.23 | 20.80 | 30.75 | 27.56 | 27.01 | +| SSIM | 0.837 | 0.873 | 0.914 | 0.882 | 0.796 | 0.714 | 0.911 | 0.911 | 0.861 | +| LPIPS | 0.093 | 0.055 | 0.025 | 0.070 | 0.103 | 0.122 | 0.029 | 0.044 | 0.068 | + + + +Ground-truth (top) and synthetic view (bottom) images generated by BioNeRF regarding four LLFF dataset’s scenes. diff --git a/docs/nerfology/methods/index.md b/docs/nerfology/methods/index.md index 99aa638d7..af0761ead 100644 --- a/docs/nerfology/methods/index.md +++ b/docs/nerfology/methods/index.md @@ -44,6 +44,7 @@ The following methods are supported in nerfstudio: PyNeRF SeaThru-NeRF Zip-NeRF + BioNeRF ``` (own_method_docs)= diff --git a/nerfstudio/configs/external_methods.py b/nerfstudio/configs/external_methods.py index ff32e9f13..7fd62636f 100644 --- a/nerfstudio/configs/external_methods.py +++ b/nerfstudio/configs/external_methods.py @@ -141,6 +141,19 @@ To enable Volinga, you must install it first by running: ) ) +# BioNeRF +external_methods.append( + ExternalMethod( + """[bold yellow]BioNeRF[/bold yellow] +For more information visit: https://docs.nerf.studio/nerfology/methods/bionerf.html + +To enable BioNeRF, you must install it first by running: + [grey]pip install git+https://github.com/Leandropassosjr/ns_bionerf[/grey]""", + configurations=[("BioNeRF", "BioNeRF. Nerfstudio implementation")], + pip_package="git+https://github.com/Leandropassosjr/ns_bionerf", + ) +) + # Instruct-GS2GS external_methods.append( ExternalMethod( From 3d61d8cbdcb66f2526d2b1a75ef20308ffafa5c1 Mon Sep 17 00:00:00 2001 From: Leandro Passos Date: Tue, 7 May 2024 04:43:10 -0300 Subject: [PATCH 3/3] Fixing typos and symbols in bionerf.md (#3126) * bionerf markdown file * method description in markdown file * inserting bionerf in index file * bionerf description * fixing image paths * including results on readme * deleting last line from bionerfs page * fixing typos in bionerf.md --- docs/nerfology/methods/bionerf.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/nerfology/methods/bionerf.md b/docs/nerfology/methods/bionerf.md index c5d3c5295..5ab344522 100644 --- a/docs/nerfology/methods/bionerf.md +++ b/docs/nerfology/methods/bionerf.md @@ -51,13 +51,13 @@ Here is an overview pipeline for BioNeRF, we will walk through each component in The first step consists of feeding two neural models simultaneously, namely $M_{\Delta}$ and $M_c$, with the camera positional information. The output of these models encodes the positional information from the input image. Although the input is the same, the neural models do not share weights and follow a different flow in the next steps. ### Cognitive Filtering -This step performs a series of operations, called \emph{filters}, that work on the embeddings coming from the previous step. There are four filters this step derives: density, color, memory, and modulation. +This step performs a series of operations, called *filters*, that work on the embeddings coming from the previous step. There are four filters this step derives: density, color, memory, and modulation. ### Memory Updating -Updating the memory requires the implementation of a mechanism capable of obliterating trivial information, which is performed using the memory filter (Step 3.1 in Figure~\ref{f.bionerf}). Fist, one needs to compute a signal modulation **$\mu$**, for further introducing new experiences in the memory **$\Psi$** through the modulating variable **$\mu$** using a $\textit{tanh}$ function (Step 3.2 in the figure). +Updating the memory requires the implementation of a mechanism capable of obliterating trivial information, which is performed using the memory filter (Step 3.1 in the figure). Fist, one needs to compute a signal modulation **$\mu$**, for further introducing new experiences in the memory **$\Psi$** through the modulating variable **$\mu$** using a $\textit{tanh}$ function (Step 3.2 in the figure). ### Contextual Inference -This step is responsible for adding contextual information to BioNeRF. Two new embeddings are generate, i.e., **${h}^{\prime}_\Delta$** and **${h}^{\prime}_c$** based on density and color filters, respectively (Step 4 in the figure), which further feed two neural models, i.e., $M^\prime_\Delta$ and $M^{\prime}$. Subsequently, $M^{\prime}_\Delta$ outputs the volume density, while color information is predicted by $M^{\prime}_c$, further used to compute the final predicted pixel information and the loss function. +This step is responsible for adding contextual information to BioNeRF. Two new embeddings are generated, i.e., **${h}^{\prime}_\Delta$** and **${h}^{\prime}_c$** based on density and color filters, respectively (Step 4 in the figure), which further feed two neural models, i.e., $M^\prime_\Delta$ and $M^\prime_c$. Subsequently, $M^\prime_\Delta$ outputs the volume density, while color information is predicted by $M^\prime_c$, further used to compute the final predicted pixel information and the loss function. ## Results