diff --git a/Dockerfile b/Dockerfile index f17fe997c..73cb6fa2c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,7 @@ -ARG CUDA_VERSION=11.8.0 +# Default to CUDA 11.7 (torch 2.0.1+cu117) to match the known-working environment. +# cu117 runs on older NVIDIA drivers (r450+) via CUDA-11 minor-version compat; +# cu118 requires a newer driver. Override with --build-arg CUDA_VERSION=... . +ARG CUDA_VERSION=11.7.1 ARG OS_VERSION=22.04 ARG USER_ID=1000 # Define base image. @@ -11,9 +14,18 @@ LABEL org.opencontainers.image.licenses = "Apache License 2.0" LABEL org.opencontainers.image.base.name="docker.io/library/nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${OS_VERSION}" # Variables used at build time. -## CUDA architectures, required by Colmap and tiny-cuda-nn. -## NOTE: All commonly used GPU architectures are included and supported here. To speedup the image build process remove all architectures but the one of your explicit GPU. Find details here: https://developer.nvidia.com/cuda-gpus (8.6 translates to 86 in the line below) or in the docs. -ARG CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37 +## CUDA architectures for tiny-cuda-nn. +## NOTE: sm_89 (Ada) and sm_90 (Hopper) require CUDA >= 11.8, so they are omitted +## here to stay compatible with the default cu117 toolkit. To speed up the build, +## keep only your GPU's architecture (find it at https://developer.nvidia.com/cuda-gpus, +## e.g. 8.6 -> 86). Add 89;90 back only if you also bump CUDA_VERSION to >= 11.8.0. +ARG CUDA_ARCHITECTURES=86;80;75;70;61;52;37 + +# Cap parallel compile jobs for the from-source builds (glog/ceres/colmap/tcnn). +# Ceres+Eigen template compilation is memory-heavy: `make -j$(nproc)` on a +# many-core host can exhaust RAM and crash GCC (segfault ICE). Raise this only +# if the machine has plenty of RAM, e.g. --build-arg MAKE_JOBS=$(nproc). +ARG MAKE_JOBS=4 # Set environment variables. ## Set non-interactive to prevent asking for user inputs blocking image creation. @@ -62,44 +74,17 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* -# Install GLOG (required by ceres). -RUN git clone --branch v0.6.0 https://github.com/google/glog.git --single-branch && \ - cd glog && \ - mkdir build && \ - cd build && \ - cmake .. && \ - make -j `nproc` && \ - make install && \ - cd ../.. && \ - rm -rf glog -# Add glog path to LD_LIBRARY_PATH. -ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" - -# Install Ceres-solver (required by colmap). -RUN git clone --branch 2.1.0 https://ceres-solver.googlesource.com/ceres-solver.git --single-branch && \ - cd ceres-solver && \ - git checkout $(git describe --tags) && \ - mkdir build && \ - cd build && \ - cmake .. -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF && \ - make -j `nproc` && \ - make install && \ - cd ../.. && \ - rm -rf ceres-solver - -# Install colmap. -RUN git clone --branch 3.8 https://github.com/colmap/colmap.git --single-branch && \ - cd colmap && \ - mkdir build && \ - cd build && \ - cmake .. -DCUDA_ENABLED=ON \ - -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES} && \ - make -j `nproc` && \ - make install && \ - cd ../.. && \ - rm -rf colmap +# NOTE: The COLMAP/Ceres/GLOG stack (and pycolmap/hloc/pyceres/pixel-perfect-sfm +# below) from the upstream Nerfstudio image has been removed. PanDORA obtains +# camera poses from OpenSFM (run outside this container) and trains on already +# processed *_ns scenes, so `ns-train PanDORA` never uses COLMAP-based SfM. +# Dropping it avoids a long, memory-heavy C++ build and shrinks the image. # Create non root user and setup environment. +# Re-declare USER_ID inside the build stage: ARGs declared before FROM are only +# in scope for the FROM line, so ${USER_ID} would otherwise be empty in RUN steps. +# (Placed here, after the cached colmap build, to avoid invalidating that layer.) +ARG USER_ID=1000 RUN useradd -m -d /home/user -g root -G sudo -u ${USER_ID} user RUN usermod -aG sudo user # Set user password @@ -118,37 +103,40 @@ SHELL ["/bin/bash", "-c"] # Upgrade pip and install packages. RUN python3.10 -m pip install --upgrade pip setuptools pathtools promise pybind11 # Install pytorch and submodules +# Re-declare CUDA_VERSION in the build stage (same before-FROM scope issue as USER_ID). +ARG CUDA_VERSION RUN CUDA_VER=${CUDA_VERSION%.*} && CUDA_VER=${CUDA_VER//./} && python3.10 -m pip install \ torch==2.0.1+cu${CUDA_VER} \ torchvision==0.15.2+cu${CUDA_VER} \ --extra-index-url https://download.pytorch.org/whl/cu${CUDA_VER} + +# CRITICAL: pin torch/torchvision/numpy for ALL later pip installs via a global +# constraints file + pip.conf. nerfstudio (torch>=1.13.1) and several extras have +# unbounded/newer torch requirements; without this they upgrade torch to the latest +# CUDA-13 build, which the host driver (CUDA 11.x) cannot run -> CUDA unavailable. +# - pyequilib==0.5.6: newer pyequilib (>=0.6) dropped the Equi2Pers class that +# nerfstudio's equirect_utils imports; 0.5.6 is the last version that keeps it. +RUN CUDA_VER=${CUDA_VERSION%.*} && CUDA_VER=${CUDA_VER//./} && \ + printf 'torch==2.0.1+cu%s\ntorchvision==0.15.2+cu%s\nnumpy<2\npyequilib==0.5.6\n' "$CUDA_VER" "$CUDA_VER" > /home/user/pip-constraints.txt && \ + mkdir -p /home/user/.config/pip && \ + printf '[global]\nextra-index-url = https://download.pytorch.org/whl/cu%s\n' "$CUDA_VER" > /home/user/.config/pip/pip.conf +ENV PIP_CONSTRAINT=/home/user/pip-constraints.txt + # Install tynyCUDNN (we need to set the target architectures as environment variable first). ENV TCNN_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES} -RUN python3.10 -m pip install git+https://github.com/NVlabs/tiny-cuda-nn.git@v1.6#subdirectory=bindings/torch +# Cap ninja/nvcc parallelism (same OOM concern as the ceres build above). +ENV MAX_JOBS=${MAKE_JOBS} +# tiny-cuda-nn's setup.py imports torch at build time. Modern pip builds wheels in +# an isolated env that lacks torch, so use --no-build-isolation to see the torch +# installed above; ninja is required to compile the CUDA bindings. +# Note: tiny-cuda-nn has no `v1.7` git tag (that version was only ever on master); +# v1.6 is the last real tag, builds on CUDA 11.7, and matches nerfstudio 0.3.2. +RUN python3.10 -m pip install ninja wheel && \ + python3.10 -m pip install --no-build-isolation \ + git+https://github.com/NVlabs/tiny-cuda-nn.git@v1.6#subdirectory=bindings/torch -# Install pycolmap, required by hloc. -RUN git clone --branch v0.4.0 --recursive https://github.com/colmap/pycolmap.git && \ - cd pycolmap && \ - python3.10 -m pip install . && \ - cd .. - -# Install hloc master (last release (1.3) is too old) as alternative feature detector and matcher option for nerfstudio. -RUN git clone --branch master --recursive https://github.com/cvg/Hierarchical-Localization.git && \ - cd Hierarchical-Localization && \ - python3.10 -m pip install -e . && \ - cd .. - -# Install pyceres from source -RUN git clone --branch v1.0 --recursive https://github.com/cvg/pyceres.git && \ - cd pyceres && \ - python3.10 -m pip install -e . && \ - cd .. - -# Install pixel perfect sfm. -RUN git clone --branch v1.0 --recursive https://github.com/cvg/pixel-perfect-sfm.git && \ - cd pixel-perfect-sfm && \ - python3.10 -m pip install -e . && \ - cd .. +# (pycolmap / hloc / pyceres / pixel-perfect-sfm removed — see note above. These +# are COLMAP-based SfM feature matchers that the PanDORA pipeline does not use.) RUN python3.10 -m pip install omegaconf # Copy nerfstudio folder and give ownership to user. @@ -162,6 +150,59 @@ RUN cd nerfstudio && \ python3.10 -m pip install -e . && \ cd .. +# --------------------------------------------------------------------------- +# PanDORA additions (on top of the base Nerfstudio image) +# --------------------------------------------------------------------------- + +# Blender (pinned LTS). hdr_blender.py / render_all.py invoke `blender --background`, +# so we need the Blender binary on PATH (not the pip `bpy` module). +USER root +ARG BLENDER_VERSION=3.6.5 +ARG BLENDER_MAJOR=3.6 +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + libxi6 libxxf86vm1 libxfixes3 libxrender1 libgl1 libsm6 xz-utils unzip && \ + rm -rf /var/lib/apt/lists/* && \ + wget -q https://download.blender.org/release/Blender${BLENDER_MAJOR}/blender-${BLENDER_VERSION}-linux-x64.tar.xz -O /tmp/blender.tar.xz && \ + mkdir -p /opt/blender && \ + tar -xf /tmp/blender.tar.xz -C /opt/blender --strip-components=1 && \ + rm /tmp/blender.tar.xz && \ + ln -s /opt/blender/blender /usr/local/bin/blender + +# PanDORA-specific Python dependencies (as the non-root user). +USER ${USER_ID} +RUN python3.10 -m pip install --no-cache-dir \ + scikit-surgerycore \ + pydub \ + skylibs \ + piq \ + OpenEXR \ + Imath + +# equilib: provided by pyequilib==0.5.6, pinned in the constraints file above and +# installed with nerfstudio. We deliberately do NOT use the repo's vendored +# equilib/ folder: it is git-ignored (absent on a fresh clone) and, because the +# repo root is on PYTHONPATH, a folder literally named `equilib/` shadows the real +# package as an empty namespace ("Equi2Pers unknown location"). + +# NOTE: lang-segment-anything is intentionally NOT installed. Current lang-sam +# requires torch>=2.3 (needs a newer driver than this cu117 build targets) and is +# only used for optional mask generation — the released dataset scenes already +# ship masks. Install a torch-2.0-compatible lang-sam separately if you need it. + +# Pin runtime dependency versions that the floating requirements otherwise break: +# - numpy<2: opencv-python==4.6.0.66 (and the torch 2.0.1 stack) are compiled +# against numpy 1.x; some extras above pull numpy 2.x, breaking cv2 import. +# - viser==0.0.16: nerfstudio 0.3.2's viewer_beta uses the old viser GUI API +# (GuiHandle, ...) that newer viser (>=0.1.0) removed. +# Kept last so earlier installs can't override these pins. +RUN python3.10 -m pip install --no-cache-dir "numpy<2" "viser==0.0.16" + +# Make the repo root importable so `lantern_scripts` (used by nerfstudio's lantern +# processors and PanDORA config) resolves without a manual `export PYTHONPATH`. +# The host repo is mounted onto this same path at runtime. +ENV PYTHONPATH=/home/user/nerfstudio + # Change working directory WORKDIR /workspace diff --git a/README.md b/README.md index 71a72d6b1..3eb331a54 100644 --- a/README.md +++ b/README.md @@ -1,323 +1,107 @@ -# nerfstudio-HDR — How it works? +# PanDORA: Casual HDR Radiance Acquisition of Indoor Scenes for Image-based Lighting -This repository supports two HDR NeRF pipelines that share the same capture, -pre-processing, and evaluation workflow, and differ only in the training and -metrics steps: +Official code release for the paper **"PanDORA: Casual HDR Radiance Acquisition +of Indoor Scenes for Image-based Lighting"** (ICCP 2026). -- **PanDORA** — two-stage training with fast/well exposure alignment. -- **HDR-Nerfacto (`hdr-nerfacto`)** — single-stage training. +- 📄 Paper: https://arxiv.org/abs/2407.06150 +- 🌐 Project page: https://lvsn.github.io/pandora/ -Steps that are identical for both methods are written once below. Where the two -methods diverge (setup branch, training, metrics) both variants are shown. +PanDORA reconstructs an HDR radiance field of an indoor scene from two 360° +videos captured at different exposures. This repo lets you download a scene and +train the model with a single Docker environment. -## 0. Setup +Built on [Nerfstudio](https://github.com/nerfstudio-project/nerfstudio). The +`hdr-nerfacto` method is our implementation of +[HDR-NeRF](https://xhuangcv.github.io/hdr-nerf/) (Huang et al., CVPR 2022). -> Tip: Open this folder in the VS Code **Dev Container** extension. +--- -``` -tmux new -s nerf_studio +## Quick start -docker run -it --rm -v /gel/usr/{USER_NAME}:/mnt/workspace/ -v /home-local2/{USER_NAME}.extra.nobkp/:/mnt/data/ --gpus '"device=0"' -p 8008:8008 -p 6006:6006 lantern_docker:latest /bin/bash +**Requirements:** an NVIDIA GPU, [Docker](https://docs.docker.com/get-docker/), +and the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). -cd /mnt/workspace/lantern/nerfstudio-HDR +Everything runs inside a Docker image that this repo builds for you. -git clone https://github.com/MrKarimiD/nerfstudio-HDR.git -conda activate nerfstudio-HDR -export PYTHONPATH=$pwd:$PYTHONPATH -pip install scikit-surgerycore pydub skylibs piq OpenEXR Imath equilib +### 1. Build the image and open a shell + +```bash +docker/build_and_run.sh --shell ``` -**Select the branch for your method:** +The first build takes a while (it compiles a few components). After it finishes +you land in a shell inside the container. All the commands below run there. -- **HDR-Nerfacto** — check out the `hdr-nerfacto` branch: - ``` - git checkout -t remotes/origin/hdr-nerfacto - ``` -- **Pandora** — use the default branch (no checkout needed). +### 2. Download a scene -Then finish the install: - -``` -pip install -e . -ns-install-cli - -pip install -U git+https://github.com/luca-medeiros/lang-segment-anything.git +```bash +docker/download_scenes.sh --list # see all available scenes +docker/download_scenes.sh --dest /data meeting_room # download one (~5 GB) ``` -If `lang-sam` is not installed, do the following: -``` -cd ../lang-segment-anything -pip install . -cd ../nerfstudio-HDR +### 3. Prepare the scene + +```bash +ns-process-data lantern-openSFM \ + --data /data/meeting_room/data/ \ + --output-dir /data/meeting_room/meeting_room_ns/ \ + --metadata /data/meeting_room/sfm/reconstruction.json ``` -If there is an issue with `equilib`, do the following: -``` -pip uninstall equilib -cd ../equilib -python setup.py develop -cd ../nerfstudio-HDR +### 4. Train + +```bash +DATASET_DIR=/data/meeting_room/meeting_room_ns/ docker/run_pandora_pipeline.sh ``` -## 1. Data acquisition & pre-processing +Results are written under `outputs/pandora/PanDORA/`. -1. Setup the apparatus (stick with two Ricoh cameras) - > Cameras: - > - > **Camera 1**: left, well-exposed, serial: YN14100695 \ - > **Camera 3**: right, fast-exposed (under-exposed), serial: YN14111000 +> Tip: for a quick test run, add `STEP1_ITERS=200 STEP2_ITERS=200` before the +> command to train only a few iterations. - - To connect the other camera: - - Turn wifi off - - Close the app - - Turn wifi on - - Open the app - - Connect the other camera - - Camera setup: - - Install Ricoh Theta app - - Connect the camera to the app - - Change camera parameters: select manual (bottom right) - - First time: turn on "CT Settings" in Shooting settings - - Apreture: 2.1 - - Shutter speed: determined below - - ISO: 800 - - WB: 3500 - - Make sure the cameras point up to avoid having lights in the seam. - - Make sure to film up and down to get good coverage +--- -2. First capture with well-exposed cameras. - - Change shutter speed to get a well-exposed image. **Note the shutter speed.** - - Start the video on one camera. Connect to the other camera following the steps above and start the video on the second camera. - - Clap at the beginning and at the end to use for synchronization. - - Go around the scene with the two cameras well-exposed. +## Advanced -3. Second capture with Camera 1 (left) well-exposed and Camera 2 (right) fast-exposed. Change shutter speed of Camera 2 to 1/25000 (fastest exposure) and repeat the steps above +The steps above cover the common case. For the full workflow — capturing your +own scenes, running OpenSFM, and computing metrics — see: - If calculating **METRICS**: Capture GT HDR brakets - - Go to settings and change the camera mode shooting method to multiple brackets - - Set the camera parameters: - - Apreture: 2.1 - - Shutter speed: Calculate the 11 shutter speeds and select the closest ones on the app in order from smallest to hightes - ``` - python lantern_scripts/calculate_exposures.py --min 0.00004 --max 0.004 - ``` - - ISO: 800 - - WB: 3500 - - Take 10 different locations +- [`README-pandora.md`](README-pandora.md) — the PanDORA pipeline in detail. +- [`README-hdrnerfacto.md`](README-hdrnerfacto.md) — the HDR-Nerfacto pipeline. -4. Import the files on computer. If on a mac, use Ricoh Theta File Transfer for Mac +--- -5. Get the equirectangular videos by processing them with the Ricoh Theta computer app. - - Drag and drop the video file into the app window. - - Make sure both boxes are unchecked. - - Name files as fallows - - Camera 1: - - First capture: left_sfm - - Second capture: left_e1 - - Camera 2 (fast exposed): - - First capture: right_sfm - - Second capture: right_e2 +## Citation - If calculating **METRICS**: Group GT brackets under GT folder like so - GT/ - GT1/ - 001.jpg - 002.jpg - ... - GT2/ - ... - ... - -6. Transfer files on lab machine. - - ``` - scp -r /path/to/source/file /path/to/destination/ - ``` - -7. Process the videos for OpenSFM. Input the right shutter speed noted above. - - If not calculating metrics: - ``` - python lantern_scripts/process_videos_for_sfm.py --input_dir /mnt/data/scene/ --shutter_speed 0.004 - ``` - - If calculating **METRICS**: - ``` - python lantern_scripts/process_videos_for_sfm.py --input_dir /mnt/data/scene/ --shutter_speed 0.004 --gt - ``` - - Resize GT_exr and GT_jpg images to 3840x1920 with GNU Image Manipulation Program - - Go to Image > Scale Image - - Transfer those back on lab machine in th GT folder of the scene naming them GT_jpg_small and GT_exr_small - - Add GT_jpg images into sfm/images/ with their appropriate mask (checkout stick_masks folder) - -## 2. Process data using OpenSFM - -``` -bin/opensfm_run_all /mnt/data/scene/sfm/ +```bibtex +@misc{dastjerdi2025pandoracasualhdrradiance, + title = {PanDORA: Casual HDR Radiance Acquisition of Indoor Scenes for Image-based Lighting}, + author = {Mohammad Reza Karimi Dastjerdi and Dominique Tanguay-Gaudreau and Frédéric Fortier-Chouinard and Yannick Hold-Geoffroy and Nima Kalantari and Jean-François Lalonde}, + year = {2025}, + eprint = {2407.06150}, + archivePrefix = {arXiv}, + primaryClass = {cs.CV}, + url = {https://arxiv.org/abs/2407.06150} +} ``` -To view results: +This work builds on Nerfstudio and, for `hdr-nerfacto`, HDR-NeRF — please cite +them too: + +```bibtex +@inproceedings{nerfstudio, + title = {Nerfstudio: A Modular Framework for Neural Radiance Field Development}, + author = {Tancik, Matthew and Weber, Ethan and Ng, Evonne and Li, Ruilong and Yi, Brent and Kerr, Justin and Wang, Terrance and Kristoffersen, Alexander and Austin, Jake and Salahi, Kamyar and Ahuja, Abhik and McAllister, David and Kanazawa, Angjoo}, + year = 2023, + booktitle = {ACM SIGGRAPH 2023 Conference Proceedings}, + series = {SIGGRAPH '23} +} + +@inproceedings{huang2022hdr, + title = {HDR-NeRF: High Dynamic Range Neural Radiance Fields}, + author = {Huang, Xin and Zhang, Qi and Feng, Ying and Li, Hongdong and Wang, Xuan and Wang, Qing}, + booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, + pages = {18398--18408}, + year = {2022} +} ``` -python3 viewer/server.py -d /mnt/data/scene/sfm/ -``` -Check that the camera positions make sense and that the number of clusters is low. - -## 3. Process data for NeRF - -``` -ns-process-data lantern-openSFM --data /mnt/data/scene/data/ --output-dir /mnt/data/scene/scene_ns/ --metadata /mnt/data/scene/sfm/reconstruction.json -``` - -For processing existing dataset that used colmap for camera positions: -``` -ns-process-data images --data /mnt/data/garden/images --output-dir /mnt/data/garden_processed --skip-colmap --skip-image-processing --colmap-model-path /mnt/data/garden/sparse/0 -``` - -## 4. Train NeRF - -Pick the method you set up in Step 0. - -### Option A — PanDORA - -1. Run step 1 of PanDORA. - - ``` - ns-train PanDORA --data /mnt/data/scene/scene_ns/ --viewer.websocket-port 8008 --pipeline.datamanager.train-num-images-to-sample-from 1800 --pipeline.model.lantern_steps 1 --pipeline.datamanager.pixel-sampler.lantern_steps 1 --max-num-iterations 60000 - ``` - - 1.1. For better alignment between the well and the fast exposed images, run the following command. - ``` - python lantern_scripts/align_fast_to_well.py --input_dir /mnt/data/scene/scene_ns/ --config outputs/scene_ns/PanDORA/2024-08-07_143013/config.yml - ``` - -2. Run step 2 of PanDORA. - - ``` - ns-train PanDORA --data /mnt/data/scene/scene_ns/ --viewer.websocket-port 8008 --pipeline.datamanager.train-num-images-to-sample-from 1800 --pipeline.model.lantern_steps 2 --pipeline.datamanager.pixel-sampler.lantern_steps 2 --load-dir /mnt/workspace/lantern/nerfstudio-HDR/outputs/scene_ns/PanDORA/2024-06-10_155559/nerfstudio_models --pipeline.model.apply_mu_law False --max-num-iterations 120000 - ``` - -### Option B — HDR-Nerfacto (`hdr-nerfacto`) - -1. Run hdr-nerfacto. - - ``` - ns-train hdr-nerfacto --data /mnt/data/nerfstudio_ds/real_data/amphitheatre_1112_2/amphitheatre_ns/ --viewer.websocket-port 8008 --pipeline.datamanager.train-num-images-to-sample-from 1800 --max-num-iterations 120000 --viewer.websocket-port 8008 - ``` - -## 5. View results - -To use the interface: -``` -ns-viewer --load-config outputs/scene_ns/PanDORA/2024-06-06_191338/config.yml --viewer.websocket-port 8008 -``` -- For the right side up: - - Go to SCENE sub-menu. - - Clic on RESET UP DIRECTION. -- To render a video with the interface command: - - Go to RENDER sub-menu. - - Clic on ADD CAMERA to manualy set camera view points. - - Clic on RENDER to get the command. - ``` - ns-render camera-path --load-config outputs/scene_ns/PanDORA/2024-06-06_191338/config.yml --camera-path-filename /mnt/data/scene/scene_ns/camera_paths/2024-06-06_191338.json --output-path /mnt/data/scene/renders/2024-06-06_191338.mp4 - ``` - - To render a video fast-exposed: add --rendered-output-names rgb_fast - ``` - ns-render camera-path --load-config outputs/scene_ns/PanDORA/2024-06-06_191338/config.yml --camera-path-filename /mnt/data/scene/scene_ns/camera_paths/2024-06-06_191338.json --output-path /mnt/data/scene/renders/2024-06-06_191338.mp4 --rendered-output-names rgb_fast - ``` - -To get evaluation images (well-exposed): -``` -ns-eval --load-config=outputs/scene_ns/PanDORA/2024-06-17_152246/config.yml --output-path=output.json --render_output_path=/mnt/data/scene/eval -``` - -- To get fast-exposed evaluation images, change these two lines in nerfstudio/lantern/model.py: - ``` - line 238: if batch["exposure"] != 1.0: # change to == for testing fast-exposed - line 242: return None, None # comment for testing fast-exposed - ``` - -- To get evaluation images and tensorboard: - ``` - ns-eval --load-config=outputs/scene_ns/PanDORA/2024-06-20_142413/config.yml --output-path=output.json --render_output_path=/mnt/data/scene/eval --vis viewer+tensorboard - ``` -- To load tensorboard results: - ``` - tensorboard --logdir=/path/to/file - ``` - - Connect to http://localhost:6006/ - -## 6. Calculate metrics - -The single-command convenience script differs by method: - -- **PanDORA**: - ``` - python lantern_scripts/calculate_metrics.py --input_dir /mnt/data/coffee_room2/ --checkpoint /mnt/workspace/lantern/nerfstudio-HDR/outputs/coffee_room2_ns/PanDORA/2025-01-22_194346/ --metric_name unaligned - ``` -- **HDR-Nerfacto (`hdr-nerfacto`)**: - ``` - python lantern_scripts/calculate_metrics_hdrnerfacto.py --input_dir /mnt/data/coffee_room2/ --checkpoint /mnt/workspace/lantern/nerfstudio-HDR/outputs/coffee_room2_ns/PanDORA/2025-01-22_194346/ --metric_name hdr_nerfacto - ``` - -then point #8 - -**or**, run the steps manually (identical for both methods): - -1. Get ground truth transformations: - ``` - ns-process-data lantern-GT-HDR --data /mnt/data/coffee_room2/coffee_room2_ns/ --output-dir /mnt/data/coffee_room2/metrics --metadata /mnt/data/coffee_room2/sfm/reconstruction.json --checkpoint /mnt/workspace/lantern/nerfstudio-HDR/outputs/coffee_room2_ns/PanDORA/2025-01-22_194346/ - ``` - -2. Render panos at ground truth positions for well and fast exposed: - ``` - ns-render camera-path --load-config /mnt/workspace/lantern/nerfstudio-HDR/outputs/coffee_room2_ns/PanDORA/2025-01-22_194346/config.yml --camera-path-filename /mnt/data/coffee_room2/metrics/GT_transforms.json --output-path /mnt/data/coffee_room2/metrics/renders/unaligned --output-format images - ``` - ``` - ns-render camera-path --load-config /mnt/workspace/lantern/nerfstudio-HDR/outputs/coffee_room2_ns/PanDORA/2025-01-22_194346/config.yml --camera-path-filename /mnt/data/coffee_room2/metrics/GT_transforms.json --output-path /mnt/data/coffee_room2/metrics/renders/unaligned_fast --output-format images --rendered_output_names rgb_fast - ``` - -3. Combine both well and fast exposed panos: - ``` - python lantern_scripts/combine.py --well_dir /mnt/data/coffee_room2/metrics/renders/unaligned/ --fast_dir /mnt/data/coffee_room2/metrics/renders/unaligned_fast/ --out_dir /mnt/data/coffee_room2/metrics/renders/unaligned_lin/ --experiment_location /mnt/data/coffee_room2/data/ --do_linearization - ``` - -4. Make sure the panos from renders have the same names as the GT panos. - -5. Calculate PSNR, SSIM and LPIPS: - ``` - python lantern_scripts/ldr_res.py --gt_dir /mnt/data/coffee_room2/GT/GT_exr/ --data_dir /mnt/data/coffee_room2/metrics/renders/unaligned_lin/ - ``` - -6. Generate GT and results renders with blender: - ``` - blender --background lantern_scripts/scene_new.blend -P lantern_scripts/hdr_blender.py -- /mnt/data/coffee_room2/GT/GT_exr/ /mnt/data/coffee_room2/GT/GT_exr_renders/ - ``` - ``` - blender --background lantern_scripts/scene_new.blend -P lantern_scripts/hdr_blender.py -- /mnt/data/coffee_room2/metrics/renders/unaligned_lin/ /mnt/data/coffee_room2/metrics/renders/unaligned_lin_renders/ - ``` - -7. Calculate si-RMSE, RMSE, RGB ang. and PSNR (LDR r.). Need to change values in code: - ``` - python lantern_scripts/res_table_1.py - ``` - -8. Convert exr panos to hdr: - ``` - convert GT1.exr GT1.hdr - ``` - -9. Calulate PU-PSNR, HDR-VDP and PU-SSIM following the steps in the following link: - https://github.com/darthgera123/PanoHDR-NeRF/tree/main/LANet/metrics - ``` - cd hdrvdp-3.0.6 - ``` - ``` - matlab -batch "metric('../../data/GT_meeting_room/', '../../data/results_meeting_room/')" - ``` - - ``` - cd ../pu21/matlab/examples - ``` - ``` - matlab -batch "pupsnr('../../../../data/GT_meeting_room/', '../../../../data/results_meeting_room/')" - ``` diff --git a/docker/build_and_run.sh b/docker/build_and_run.sh new file mode 100644 index 000000000..56260dec4 --- /dev/null +++ b/docker/build_and_run.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# +# Build the PanDORA docker image and run the fixed two-stage training pipeline +# inside it, with GPU access and the repo + dataset mounted at runtime. +# +# Usage: +# docker/build_and_run.sh [--rebuild] [--shell] +# +# --rebuild Rebuild the image from scratch (docker build --no-cache). +# --shell Drop into an interactive shell instead of running the pipeline. +# +# Common environment overrides: +# DATA_ROOT Host dir mounted to /data (default below) +# DATASET_DIR Container path to a *_ns scene (default below, under /data) +# STEP1_ITERS / STEP2_ITERS Lower for a quick smoke test. +set -euo pipefail + +# --- Configuration ----------------------------------------------------------- +IMAGE="${IMAGE:-pandora:latest}" +# Repo root = parent of this script's directory. +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# Host data directory that holds the processed scenes; mounted to /data. +DATA_ROOT="${DATA_ROOT:-/home-local2/${USER}.extra.nobkp/nerfstudio_ds}" +# Container-side path to the scene to train on (lives under /data). +DATASET_DIR="${DATASET_DIR:-/data/sample_scene/sample_scene_ns/}" +SHM_SIZE="${SHM_SIZE:-12gb}" + +REBUILD=0 +RUN_SHELL=0 +for arg in "$@"; do + case "$arg" in + --rebuild) REBUILD=1 ;; + --shell) RUN_SHELL=1 ;; + -h|--help) + sed -n '2,20p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + exit 0 ;; + *) echo "Unknown argument: $arg" >&2; exit 1 ;; + esac +done + +# --- Build ------------------------------------------------------------------- +# Parallel compile jobs for the from-source builds (ceres/colmap/tcnn). Keep +# modest to avoid GCC OOM/segfault on the heavy Ceres template files; raise on +# high-RAM machines, e.g. MAKE_JOBS=8 docker/build_and_run.sh. +MAKE_JOBS="${MAKE_JOBS:-4}" +# CUDA toolkit version for the base image + torch/tcnn build. Must be <= the max +# CUDA version your host NVIDIA driver supports (see `nvidia-smi`). Default 11.7.1 +# (torch 2.0.1+cu117) matches the known-working env and runs on older drivers +# (r450+). torch 2.0.1 also ships cu118 wheels if your driver is newer. +CUDA_VERSION="${CUDA_VERSION:-11.7.1}" +BUILD_ARGS=(-t "$IMAGE" -f "$REPO/Dockerfile" + --build-arg MAKE_JOBS="$MAKE_JOBS" + --build-arg CUDA_VERSION="$CUDA_VERSION" + "$REPO") +if [ "$REBUILD" -eq 1 ]; then + echo ">> Rebuilding image '$IMAGE' from scratch (--no-cache)..." + docker build --no-cache "${BUILD_ARGS[@]}" +else + echo ">> Building image '$IMAGE' (cached)..." + docker build "${BUILD_ARGS[@]}" +fi + +# --- Run --------------------------------------------------------------------- +# Mount the repo onto the same path the image installed it editable (-e .), so +# the live code is used and the editable install stays valid. +DOCKER_RUN=( + docker run --rm -it + --gpus all + --shm-size="$SHM_SIZE" + -v "$REPO":/home/user/nerfstudio + -v "$DATA_ROOT":/data + -e DATASET_DIR="$DATASET_DIR" + -e EXPERIMENT_NAME="${EXPERIMENT_NAME:-pandora}" + -e STEP1_ITERS="${STEP1_ITERS:-60000}" + -e STEP2_ITERS="${STEP2_ITERS:-120000}" + -w /home/user/nerfstudio + "$IMAGE" +) + +if [ "$RUN_SHELL" -eq 1 ]; then + echo ">> Launching interactive shell in '$IMAGE'..." + exec "${DOCKER_RUN[@]}" bash +else + echo ">> Running PanDORA pipeline in '$IMAGE' on DATASET_DIR=$DATASET_DIR ..." + exec "${DOCKER_RUN[@]}" bash docker/run_pandora_pipeline.sh +fi diff --git a/docker/download_scenes.sh b/docker/download_scenes.sh new file mode 100644 index 000000000..48c816892 --- /dev/null +++ b/docker/download_scenes.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# +# Download PanDORA dataset scenes by name. +# Scenes are hosted as individual ZIP archives (~5 GB each) at: +# https://hdrdb-public.s3.valeria.science/pandora/.zip +# See: https://lvsn.github.io/pandora/dataset/index.html +# +# Usage: +# docker/download_scenes.sh [ ...] +# docker/download_scenes.sh all +# docker/download_scenes.sh --list +# +# Options: +# --dest DIR Destination directory (default: $DATA_ROOT or /data) +# --keep-zip Keep the downloaded .zip after extracting (default: delete) +# --force Re-download / re-extract even if already present +# --list Print the available scene names and exit +# +# Downloads are resumable (curl -C -), so re-running continues an interrupted +# download. Extraction is skipped if the scene is already present unless --force. +set -euo pipefail + +BASE_URL="https://hdrdb-public.s3.valeria.science/pandora" + +# The 14 available scenes (exact zip basenames). +SCENES=( + auditorium + auditorium_dark + basement + blue_bedroom + classroom_no_windows + classroom_windows + clubhouse + coffee_room + lab_office + living_room + lobby + meeting_room + office + small_office +) + +DEST="${DATA_ROOT:-/data}" +KEEP_ZIP=0 +FORCE=0 +REQUESTED=() + +is_valid_scene() { + local s="$1" + for known in "${SCENES[@]}"; do + [ "$known" = "$s" ] && return 0 + done + return 1 +} + +print_list() { + echo "Available PanDORA scenes (${#SCENES[@]}):" + for s in "${SCENES[@]}"; do echo " - $s"; done +} + +# --- Parse arguments --------------------------------------------------------- +while [ $# -gt 0 ]; do + case "$1" in + --dest) DEST="$2"; shift 2 ;; + --keep-zip) KEEP_ZIP=1; shift ;; + --force) FORCE=1; shift ;; + --list) print_list; exit 0 ;; + -h|--help) sed -n '2,25p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + all) REQUESTED=("${SCENES[@]}"); shift ;; + -*) echo "Unknown option: $1" >&2; exit 1 ;; + *) REQUESTED+=("$1"); shift ;; + esac +done + +if [ ${#REQUESTED[@]} -eq 0 ]; then + echo "ERROR: no scene(s) specified." >&2 + echo >&2 + print_list >&2 + echo >&2 + echo "Example: docker/download_scenes.sh meeting_room coffee_room" >&2 + exit 1 +fi + +# Validate all requested names up front. +for s in "${REQUESTED[@]}"; do + if ! is_valid_scene "$s"; then + echo "ERROR: unknown scene '$s'." >&2 + echo >&2 + print_list >&2 + exit 1 + fi +done + +mkdir -p "$DEST" + +# --- Download + extract ------------------------------------------------------ +for scene in "${REQUESTED[@]}"; do + marker="$DEST/.$scene.done" + zip="$DEST/$scene.zip" + url="$BASE_URL/$scene.zip" + + echo "==================================================================" + echo "Scene: $scene" + echo " URL : $url" + echo " Dest: $DEST" + echo "==================================================================" + + if [ -f "$marker" ] && [ "$FORCE" -eq 0 ]; then + echo "Already downloaded and extracted (marker $marker exists); skipping." + echo "Use --force to re-download." + continue + fi + + echo "Downloading (resumable)..." + curl -L -f --retry 5 --retry-delay 5 -C - -o "$zip" "$url" + + echo "Extracting..." + unzip -q -o "$zip" -d "$DEST" + + touch "$marker" + + if [ "$KEEP_ZIP" -eq 0 ]; then + echo "Removing archive $zip (use --keep-zip to keep)..." + rm -f "$zip" + fi + + echo "Done: $scene" +done + +echo "==================================================================" +echo "All requested scenes are ready under: $DEST" +echo "==================================================================" diff --git a/docker/run_pandora_pipeline.sh b/docker/run_pandora_pipeline.sh new file mode 100644 index 000000000..5c3a9f1ce --- /dev/null +++ b/docker/run_pandora_pipeline.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# +# Fixed PanDORA two-stage training pipeline (README "Option A"). +# Runs inside the PanDORA docker image, but also works in an interactive shell. +# +# step 1 (lantern_steps 1) -> align_fast_to_well.py -> step 2 (lantern_steps 2) +# +# The dataset is provided via the DATASET_DIR env var (a processed *_ns scene). +# Deterministic --experiment-name / --timestamp make the step-1 outputs a known +# path that the align + step-2 commands can reference. +set -euo pipefail + +# Make the repo root importable so `lantern_scripts` (used by nerfstudio's lantern +# processors/config) resolves even if PYTHONPATH isn't already set in the env. +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +export PYTHONPATH="${REPO_ROOT}:${PYTHONPATH:-}" + +# --- Configuration (override via environment) -------------------------------- +# Processed nerfstudio scene directory (contains transforms.json, images/, ...). +DATASET_DIR="${DATASET_DIR:-/data/sample_scene/sample_scene_ns/}" +EXPERIMENT_NAME="${EXPERIMENT_NAME:-pandora}" +# Iteration counts (lower these for a quick smoke test, e.g. STEP1_ITERS=200). +STEP1_ITERS="${STEP1_ITERS:-60000}" +STEP2_ITERS="${STEP2_ITERS:-120000}" +NUM_IMAGES_TO_SAMPLE="${NUM_IMAGES_TO_SAMPLE:-1800}" + +STEP1_OUT="outputs/${EXPERIMENT_NAME}/PanDORA/step_1" +STEP2_OUT="outputs/${EXPERIMENT_NAME}/PanDORA/step_2" + +if [ ! -d "$DATASET_DIR" ]; then + echo "ERROR: DATASET_DIR does not exist: $DATASET_DIR" + echo "Set DATASET_DIR to a processed nerfstudio scene (a *_ns directory)." + exit 1 +fi + +echo "==================================================================" +echo "PanDORA two-stage training" +echo " DATASET_DIR = $DATASET_DIR" +echo " EXPERIMENT_NAME = $EXPERIMENT_NAME" +echo " step 1 iters = $STEP1_ITERS -> $STEP1_OUT" +echo " step 2 iters = $STEP2_ITERS -> $STEP2_OUT" +echo "==================================================================" + +# --- Step 1: train on two exposures (lantern step 1) ------------------------- +echo "[1/3] Training PanDORA step 1..." +ns-train PanDORA \ + --data "$DATASET_DIR" \ + --experiment-name "$EXPERIMENT_NAME" \ + --timestamp step_1 \ + --output-dir outputs \ + --pipeline.datamanager.train-num-images-to-sample-from "$NUM_IMAGES_TO_SAMPLE" \ + --pipeline.model.lantern_steps 1 \ + --pipeline.datamanager.pixel-sampler.lantern_steps 1 \ + --max-num-iterations "$STEP1_ITERS" \ + --viewer.quit-on-train-completion True + +# --- Step 2a: align fast exposure to well exposure --------------------------- +echo "[2/3] Aligning fast exposure to well exposure..." +python lantern_scripts/align_fast_to_well.py \ + --input_dir "$DATASET_DIR" \ + --config "$STEP1_OUT/config.yml" + +# --- Step 2b: train on aligned two exposures (lantern step 2) ---------------- +echo "[3/3] Training PanDORA step 2..." +ns-train PanDORA \ + --data "$DATASET_DIR" \ + --experiment-name "$EXPERIMENT_NAME" \ + --timestamp step_2 \ + --output-dir outputs \ + --pipeline.datamanager.train-num-images-to-sample-from "$NUM_IMAGES_TO_SAMPLE" \ + --pipeline.model.lantern_steps 2 \ + --pipeline.datamanager.pixel-sampler.lantern_steps 2 \ + --pipeline.model.apply_mu_law False \ + --load-dir "$STEP1_OUT/nerfstudio_models" \ + --max-num-iterations "$STEP2_ITERS" \ + --viewer.quit-on-train-completion True + +echo "==================================================================" +echo "Done. Final model: $STEP2_OUT" +echo "=================================================================="