From 630babacd6aea32999f40c181b7e8857d9a483b4 Mon Sep 17 00:00:00 2001 From: Pablo Vela Date: Sun, 12 Mar 2023 16:32:33 -0400 Subject: [PATCH] Add option to use Pixel Perfect SFM (#1554) add pixsfm option when using hloc --- .devcontainer/docker-compose.yml | 8 ++--- Dockerfile | 14 ++++++++ nerfstudio/process_data/hloc_utils.py | 48 +++++++++++++++++++++------ scripts/process_data.py | 5 +++ 4 files changed, 61 insertions(+), 14 deletions(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index de862fc1a..5cbf79cc3 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -3,10 +3,10 @@ version: "3.8" services: nerfstudio: shm_size: '12gb' - image: dromni/nerfstudio:0.1.18 - # build: - # context: .. - # dockerfile: ./Dockerfile + image: nerfstudio:latest + build: + context: .. + dockerfile: ./Dockerfile stdin_open: true tty: true environment: diff --git a/Dockerfile b/Dockerfile index 2bdb5328b..92af70932 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,6 +34,7 @@ RUN apt-get update && \ libboost-program-options-dev \ libboost-system-dev \ libboost-test-dev \ + libhdf5-dev \ libcgal-dev \ libeigen3-dev \ libflann-dev \ @@ -132,6 +133,19 @@ RUN git clone --branch master --recursive https://github.com/cvg/Hierarchical-Lo python3.10 -m pip install -e . && \ cd .. +# Install pyceres from source +RUN git clone --branch main --recursive https://github.com/cvg/pyceres.git && \ + cd pyceres && \ + python3.10 -m pip install -e . && \ + cd .. + +# Install pixel perfect sfm. +RUN git clone --branch main --recursive https://github.com/cvg/pixel-perfect-sfm && \ + cd pixel-perfect-sfm && \ + python3.10 -m pip install -e . && \ + cd .. + +RUN python3.10 -m pip install omegaconf # Copy nerfstudio folder and give ownership to user. ADD . /home/user/nerfstudio USER root diff --git a/nerfstudio/process_data/hloc_utils.py b/nerfstudio/process_data/hloc_utils.py index 0f46dd12c..0aec2376e 100644 --- a/nerfstudio/process_data/hloc_utils.py +++ b/nerfstudio/process_data/hloc_utils.py @@ -42,6 +42,13 @@ except ImportError: else: _HAS_HLOC = True +try: + from pixsfm.refine_hloc import PixSfM +except ImportError: + _HAS_PIXSFM = False +else: + _HAS_PIXSFM = True + CONSOLE = Console(width=120) @@ -58,6 +65,7 @@ def run_hloc( "superglue", "superglue-fast", "NN-superpoint", "NN-ratio", "NN-mutual", "adalam" ] = "superglue", num_matched: int = 50, + refine_pixsfm: bool = False, ) -> None: """Runs hloc on the images. @@ -75,6 +83,10 @@ def run_hloc( ) sys.exit(1) + if refine_pixsfm and not _HAS_PIXSFM: + CONSOLE.print("[bold red]Error: use refine_pixsfm, you must install pixel-perfect-sfm toolbox!!") + sys.exit(1) + outputs = colmap_dir sfm_pairs = outputs / "pairs-netvlad.txt" sfm_dir = outputs / "sparse" / "0" @@ -99,13 +111,29 @@ def run_hloc( image_options = pycolmap.ImageReaderOptions( # pylint: disable=c-extension-no-member camera_model=camera_model.value ) - reconstruction.main( - sfm_dir, - image_dir, - sfm_pairs, - features, - matches, - camera_mode=pycolmap.CameraMode.SINGLE, # pylint: disable=c-extension-no-member - image_options=image_options, - verbose=verbose, - ) + if refine_pixsfm: + sfm = PixSfM({"dense_features": {"max_edge": 1024}}) + refined, _ = sfm.reconstruction( + sfm_dir, + image_dir, + sfm_pairs, + features, + matches, + image_list=references, + camera_mode=pycolmap.CameraMode.SINGLE, # pylint: disable=c-extension-no-member + image_options=image_options, + verbose=verbose, + ) + print("Refined", refined.summary()) + + else: + reconstruction.main( + sfm_dir, + image_dir, + sfm_pairs, + features, + matches, + camera_mode=pycolmap.CameraMode.SINGLE, # pylint: disable=c-extension-no-member + image_options=image_options, + verbose=verbose, + ) diff --git a/scripts/process_data.py b/scripts/process_data.py index 570fda59e..2e08bf9a9 100644 --- a/scripts/process_data.py +++ b/scripts/process_data.py @@ -55,6 +55,8 @@ class ProcessImages: sfm_tool: Literal["any", "colmap", "hloc"] = "any" """Structure from motion tool to use. Colmap will use sift features, hloc can use many modern methods such as superpoint features and superglue matcher""" + refine_pixsfm: bool = False + """If True, runs refinement using Pixel Perfect SFM. Only works with hloc sfm_tool""" feature_type: Literal[ "any", "sift", @@ -211,6 +213,9 @@ class ProcessImages: (sfm_tool, feature_type, matcher_type) = process_data_utils.find_tool_feature_matcher_combination( self.sfm_tool, self.feature_type, self.matcher_type ) + # check that sfm_tool is hloc if using refine_pixsfm + if self.refine_pixsfm: + assert sfm_tool == "hloc", "refine_pixsfm only works with sfm_tool hloc" if sfm_tool == "colmap": colmap_utils.run_colmap(