diff --git a/.gitignore b/.gitignore index 450c4ea95..19e4e736a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,11 @@ # Ours Dual_Cameras gt_test +hdr-nerfacto-subset-out +ablations_out ablations* renders* - - # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/README-pandora.md b/README-pandora.md index 48431ee29..6db4c6515 100644 --- a/README-pandora.md +++ b/README-pandora.md @@ -2,6 +2,9 @@ ## 0. Setup + +Open this folder in VS Code Dev Container extension! + ``` tmux new -s nerf_studio @@ -275,4 +278,4 @@ or ``` ``` matlab -batch "pupsnr('../../../../data/GT_meeting_room/', '../../../../data/results_meeting_room/')" - ``` \ No newline at end of file + ``` diff --git a/experiments.md b/experiments.md new file mode 100755 index 000000000..652b78597 --- /dev/null +++ b/experiments.md @@ -0,0 +1,148 @@ + + +`` +renders/lab_downstairs_ns/2024-05-10_211317/hdr-nerfacto +`` + +No camera optim + +outputs/lab_downstairs_ns/hdr-nerfacto/2024-05-11_141957/config.yml + +Train: +``` +export SCENE=lab_downstairs_ns + +ns-train hdr-nerfacto --data data/$SCENE --pipeline.datamanager.train-num-images-to-sample-from 1800 +``` + +Eval (GT+normal): +``` +export SCENE=lab_downstairs_ns +export MODEL_NAME=hdr-nerfacto/2024-05-11_141957 + +ns-eval --load-config outputs/$SCENE/$MODEL_NAME/config.yml --output-path ablations_out/$SCENE/$MODEL_NAME/output.json --render-output-path ablations_out/$SCENE/$MODEL_NAME ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_well --output-format images --image-format png --rendered-output-names rgb ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_fast --output-format images --image-format png --rendered-output-names rgb_fast ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_hdr --output-format images --image-format exr --rendered-output-names rgb_hdr +``` + +Move (outside docker) +``` +mkdir -p /gel/usr/mokad6/ml_class/fred-out/ ; \ +cp -R /home-local2/frfoc1.extra.nobkp/nerfstudio-HDR/ablations_out /gel/usr/mokad6/ml_class/fred-out/ns-eval ; \ +cp -R /home-local2/frfoc1.extra.nobkp/nerfstudio-HDR/renders /gel/usr/mokad6/ml_class/fred-out/ns-render +``` + +1800 samples: +export SCENE=lab_downstairs_ns +export MODEL_NAME=hdr-nerfacto/2024-05-12_115558 + +1600 samples: +export SCENE=JF_office_ns +export MODEL_NAME=hdr-nerfacto/2024-05-14_152107 + + + + +Test wo CRF + +export SCENE=lab_downstairs_ns +export MODEL_NAME=hdr-nerfacto/2024-05-12_115558 + + + +## Fix pixel sampler + +1800 samples, fix pixel sampler: +export SCENE=Henry_Gagnon_blue_et_rouge_ns +export MODEL_NAME=hdr-nerfacto/2024-05-14_175753 +Exp: 0.002..... +DONE + +1800 samples, fix pixel sampler: +export SCENE=JF_office_ns +export MODEL_NAME=hdr-nerfacto/2024-05-14_182719 +Exp: 0.008 + +1800 samples, fix pixel sampler: +export SCENE=lab_downstairs_ns +export MODEL_NAME=hdr-nerfacto/2024-05-14_191020 +Exp: 0.01 + +1800 samples, fix pixel sampler: +export SCENE=meeting_room_ns +export MODEL_NAME=hdr-nerfacto/2024-05-15_203121 +Exp: 0.008 + +1800 samples, w/o CRF, fix pixel sampler: +export SCENE=lab_downstairs_ns +export MODEL_NAME=hdr-nerfacto-wo-crf/2024-05-14_194316 +Exp: 0.01 + +1800 samples, w/o CRF, clip before acc, fix pixel sampler: +export SCENE=lab_downstairs_ns +export MODEL_NAME=hdr-nerfacto-wo-crf-clip-bf-acc/2024-05-14_200611 +Exp: 0.01 + + +## Export all + +export SCENE=Henry_Gagnon_blue_et_rouge_ns ; \ +export MODEL_NAME=hdr-nerfacto/2024-05-14_175753 ; \ +ns-eval --load-config outputs/$SCENE/$MODEL_NAME/config.yml --output-path ablations_out/$SCENE/$MODEL_NAME/output.json --render-output-path ablations_out/$SCENE/$MODEL_NAME ; \ +export SCENE=JF_office_ns ; \ +export MODEL_NAME=hdr-nerfacto/2024-05-14_182719 ; \ +ns-eval --load-config outputs/$SCENE/$MODEL_NAME/config.yml --output-path ablations_out/$SCENE/$MODEL_NAME/output.json --render-output-path ablations_out/$SCENE/$MODEL_NAME ; \ +export SCENE=lab_downstairs_ns ; \ +export MODEL_NAME=hdr-nerfacto/2024-05-14_191020 ; \ +ns-eval --load-config outputs/$SCENE/$MODEL_NAME/config.yml --output-path ablations_out/$SCENE/$MODEL_NAME/output.json --render-output-path ablations_out/$SCENE/$MODEL_NAME ; \ +export SCENE=lab_downstairs_ns ; \ +export MODEL_NAME=hdr-nerfacto-wo-crf/2024-05-14_194316 ; \ +ns-eval --load-config outputs/$SCENE/$MODEL_NAME/config.yml --output-path ablations_out/$SCENE/$MODEL_NAME/output.json --render-output-path ablations_out/$SCENE/$MODEL_NAME ; \ +export SCENE=lab_downstairs_ns ; \ +export MODEL_NAME=hdr-nerfacto-wo-crf-clip-bf-acc/2024-05-14_200611 ; \ +ns-eval --load-config outputs/$SCENE/$MODEL_NAME/config.yml --output-path ablations_out/$SCENE/$MODEL_NAME/output.json --render-output-path ablations_out/$SCENE/$MODEL_NAME + +### Models trained for 60K steps + +outputs/lab_downstairs_ns/hdr-nerfacto/2024-05-18_012547/config.yml +outputs/meeting_room_ns/hdr-nerfacto/2024-05-18_020406/config.yml + +fast_exp=0.01 +export SCENE=lab_downstairs_ns ; \ +export MODEL_NAME=hdr-nerfacto/2024-05-18_012547 ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_well --output-format images --image-format png --rendered-output-names rgb ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_fast --output-format images --image-format png --rendered-output-names rgb_fast ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_hdr --output-format images --image-format exr --rendered-output-names rgb_hdr + +export SCENE=meeting_room_ns ; \ +export MODEL_NAME=hdr-nerfacto/2024-05-18_020406 ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_well --output-format images --image-format png --rendered-output-names rgb ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_fast --output-format images --image-format png --rendered-output-names rgb_fast ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_hdr --output-format images --image-format exr --rendered-output-names rgb_hdr + +fast_exp=0.004 +export SCENE=my_apartment_ns ; \ +export MODEL_NAME=hdr-nerfacto/2024-05-18_133026 ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_well --output-format images --image-format png --rendered-output-names rgb ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_fast --output-format images --image-format png --rendered-output-names rgb_fast ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_hdr --output-format images --image-format exr --rendered-output-names rgb_hdr + +fast_exp=0.004 +export SCENE=LVSN_LAB_OpenSFM +export MODEL_NAME=hdr-nerfacto/2024-05-18_143132 +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_well --output-format images --image-format png --rendered-output-names rgb ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_fast --output-format images --image-format png --rendered-output-names rgb_fast ; \ +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_hdr --output-format images --image-format exr --rendered-output-names rgb_hdr + + +### Ablations 60K + + +export SCENE=lab_downstairs_ns ; \ +export MODEL_NAME=hdr-nerfacto-wo-crf/2024-05-18_171020 +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_hdr --output-format images --image-format exr --rendered-output-names rgb_hdr + +export SCENE=lab_downstairs_ns ; \ +export MODEL_NAME=hdr-nerfacto-wo-crf-clip-bf-acc/2024-05-18_174929 +ns-render camera-path --load-config outputs/$SCENE/$MODEL_NAME/config.yml --camera-path-filename data/$SCENE/GT_transforms.json --output-path renders/$SCENE/${MODEL_NAME}_hdr --output-format images --image-format exr --rendered-output-names rgb_hdr diff --git a/lantern_scripts/generate_small_subset_hdr_nerf.py b/lantern_scripts/generate_small_subset_hdr_nerf.py new file mode 100644 index 000000000..45133ed6d --- /dev/null +++ b/lantern_scripts/generate_small_subset_hdr_nerf.py @@ -0,0 +1,148 @@ +import argparse +import json +import math +import os +import re +import shutil +from copy import deepcopy +from pathlib import Path + +import cv2 +import numpy as np + + +def load_from_json(filename: Path): + """Load a dictionary from a JSON filename. + + Args: + filename: The filename to load from. + """ + assert filename.suffix == ".json" + with open(filename, encoding="UTF-8") as file: + return json.load(file) + +def is_mask_all_valid(mask_path): + mask = cv2.imread(mask_path, cv2.IMREAD_GRAYSCALE) + return np.all(mask > 200) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--input_json', type=str, required=True) + parser.add_argument('--output_dir', type=str, required=True) + parser.add_argument('--train_split_fraction', type=float, default= 0.60) + parser.add_argument('--eval_split_fraction', type=float, default= 0.01) + parser.add_argument('--min_frame_number', type=int, default=100) + parser.add_argument('--frames_per_direction', type=int, default=100) + args = parser.parse_args() + + assert os.path.isfile(args.input_json) and args.input_json.endswith('.json'), 'The input JSON file is missing!' + + os.makedirs(args.output_dir, exist_ok=True) + + transforms_data = load_from_json( Path(args.input_json) ) + exposures_data = load_from_json( Path(args.input_json).parent / 'exposures.json' ) + + + selected_left_frames = [] + selected_right_frames = [] + new_exposures = dict() + + os.makedirs(os.path.join(args.output_dir, 'images'), exist_ok=True) + os.makedirs(os.path.join(args.output_dir, 'masks'), exist_ok=True) + for i, frame in enumerate(transforms_data['frames']): + file_path = frame['file_path'] # e.g. images/left_e1_output_0010_0_linear.exr (frame num is 0010) + mask_path = str(Path(args.input_json).parent / frame['mask_path']) # e.g. masks/left_e1_output_0010_0.png + # get frame number + frame_num = int(re.search(r'(\d{4})', file_path).group(0)) + direction = re.search(r'(left|right)', file_path, re.IGNORECASE).group(0).lower() + + if int(frame_num) >= args.min_frame_number: + if direction == 'left': + if len(selected_left_frames) < args.frames_per_direction and is_mask_all_valid(mask_path): + selected_left_frames.append(frame) + + shutil.copy(str(Path(args.input_json).parent / file_path), os.path.join(args.output_dir, file_path)) + shutil.copy(str(Path(args.input_json).parent / file_path.replace('_linear.exr', '.png')), os.path.join(args.output_dir, file_path.replace('_linear.exr', '.png'))) + shutil.copy(mask_path, os.path.join(args.output_dir, frame['mask_path'])) + shutil.copy(mask_path.replace('.png', '_saturation_mask.png'), os.path.join(args.output_dir, frame['mask_path'].replace('.png', '_saturation_mask.png'))) + + new_exposures[file_path] = exposures_data[file_path] + else: + if len(selected_right_frames) < args.frames_per_direction and is_mask_all_valid(mask_path): + selected_right_frames.append(frame) + + shutil.copy(str(Path(args.input_json).parent / file_path), os.path.join(args.output_dir, file_path)) + shutil.copy(str(Path(args.input_json).parent / file_path.replace('_linear.exr', '.png')), os.path.join(args.output_dir, file_path.replace('_linear.exr', '.png'))) + shutil.copy(mask_path, os.path.join(args.output_dir, frame['mask_path'])) + shutil.copy(mask_path.replace('.png', '_saturation_mask.png'), os.path.join(args.output_dir, frame['mask_path'].replace('.png', '_saturation_mask.png'))) + + new_exposures[file_path] = exposures_data[file_path] + + + new_transforms_data = {} + all_keys = transforms_data.keys() + for key in all_keys: + new_transforms_data[key] = transforms_data[key] + + new_transforms_data['frames'] = selected_left_frames + selected_right_frames + + num_images_per_direction = len(selected_left_frames) + + num_train_images = math.ceil(num_images_per_direction * args.train_split_fraction) + num_eval_test_images = num_images_per_direction - num_train_images + i_all = np.arange(num_images_per_direction) + i_train = np.linspace( + 0, num_images_per_direction - 1, num_train_images, dtype=int + ) # equally spaced training images starting and ending at 0 and num_images-1 + i_eval_test = np.setdiff1d(i_all, i_train) # eval images are the remaining images + assert len(i_eval_test) == num_eval_test_images + + num_eval_images = math.ceil(num_images_per_direction * args.eval_split_fraction) + num_test_images = num_eval_test_images - num_train_images + i_eval = np.linspace( + 0, num_eval_test_images - 1, num_eval_images, dtype=int + ) # equally spaced training images starting and ending at 0 and num_images-1 + i_eval = i_eval_test[i_eval] + i_test = np.setdiff1d(i_eval_test, i_eval) # eval images are the remaining images + assert len(i_eval) == num_eval_images + + hdr_nerf_transforms_data = dict() + hdr_nerf_exposures_data = dict() + for split, indices in zip(("train", "val", "test"), (i_train, i_eval, i_test)): + hdr_nerf_transforms_data[split] = deepcopy(transforms_data) + hdr_nerf_transforms_data[split]["frames"] = [selected_left_frames[i] for i in indices] + [selected_right_frames[i] for i in indices] + hdr_nerf_exposures_data[split] = {k: v for k, v in new_exposures.items() if k in [frame["file_path"] for frame in hdr_nerf_transforms_data[split]["frames"]]} + + with open(os.path.join(args.output_dir, f'transforms_{split}.json'), "w") as outfile: + json.dump(hdr_nerf_transforms_data[split], outfile, indent=4) + + with open(os.path.join(args.output_dir, f'exposure_{split}.json'), "w") as outfile: + json.dump(hdr_nerf_exposures_data[split], outfile, indent=4) + + # has_split_files_spec = any(f"{split}_filenames" in meta for split in ("train", "val", "test")) + + train_list = [] + for i in i_train: + train_list.append(selected_left_frames[i]["file_path"]) + train_list.append(selected_right_frames[i]["file_path"]) + new_transforms_data["train_filenames"] = train_list + + test_list = [] + for i in i_test: + test_list.append(selected_left_frames[i]["file_path"]) + test_list.append(selected_right_frames[i]["file_path"]) + new_transforms_data["test_filenames"] = test_list + + eval_list = [] + for i in i_eval: + eval_list.append(selected_left_frames[i]["file_path"]) + eval_list.append(selected_right_frames[i]["file_path"]) + new_transforms_data["val_filenames"] = eval_list + + with open(os.path.join(args.output_dir, 'transforms.json'), "w") as outfile: + json.dump(new_transforms_data, outfile, indent=4) + + with open(os.path.join(args.output_dir, 'exposures.json'), "w") as outfile: + json.dump(new_exposures, outfile, indent=4) + + print("All done!") diff --git a/linearize-ricoh-z10-wb-5000.json b/linearize-ricoh-z10-wb-5000.json deleted file mode 100644 index 680d3f147..000000000 --- a/linearize-ricoh-z10-wb-5000.json +++ /dev/null @@ -1 +0,0 @@ -{"B": [46.82997512817383, 0.11160952597856522, 0.002294188365340233], "G": [41.67820739746094, 0.09581712633371353, 0.0028952741995453835], "R": [37.4744873046875, 0.22809027135372162, 0.002323441905900836], "type": "deg2"} \ No newline at end of file diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index 940c6037b..15ee77809 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -20,29 +20,33 @@ from __future__ import annotations from collections import OrderedDict from typing import Dict -from dataclasses import dataclass, field import tyro from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig from nerfstudio.configs.base_config import ViewerConfig from nerfstudio.configs.external_methods import get_external_methods - -from nerfstudio.data.datamanagers.random_cameras_datamanager import RandomCamerasDataManagerConfig -from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager, VanillaDataManagerConfig - +from nerfstudio.data.datamanagers.base_datamanager import ( + VanillaDataManager, + VanillaDataManagerConfig, +) +from nerfstudio.data.datamanagers.random_cameras_datamanager import ( + RandomCamerasDataManagerConfig, +) from nerfstudio.data.dataparsers.blender_dataparser import BlenderDataParserConfig from nerfstudio.data.dataparsers.dnerf_dataparser import DNeRFDataParserConfig -from nerfstudio.data.dataparsers.instant_ngp_dataparser import InstantNGPDataParserConfig +from nerfstudio.data.dataparsers.instant_ngp_dataparser import ( + InstantNGPDataParserConfig, +) from nerfstudio.data.dataparsers.nerfstudio_dataparser import NerfstudioDataParserConfig -from nerfstudio.data.dataparsers.phototourism_dataparser import PhototourismDataParserConfig +from nerfstudio.data.dataparsers.phototourism_dataparser import ( + PhototourismDataParserConfig, +) from nerfstudio.data.dataparsers.sdfstudio_dataparser import SDFStudioDataParserConfig from nerfstudio.data.dataparsers.sitcoms3d_dataparser import Sitcoms3DDataParserConfig from nerfstudio.data.datasets.depth_dataset import DepthDataset from nerfstudio.data.datasets.sdf_dataset import SDFDataset from nerfstudio.data.datasets.semantic_dataset import SemanticDataset -from nerfstudio.data.datasets.hdr_dataset import HDRInputDataset - from nerfstudio.engine.optimizers import AdamOptimizerConfig, RAdamOptimizerConfig from nerfstudio.engine.schedulers import ( CosineDecaySchedulerConfig, @@ -52,6 +56,8 @@ from nerfstudio.engine.schedulers import ( from nerfstudio.engine.trainer import TrainerConfig from nerfstudio.field_components.temporal_distortions import TemporalDistortionKind from nerfstudio.fields.sdf_field import SDFFieldConfig +from nerfstudio.hdr_nerfacto.config import get_hdr_nerfacto_config +from nerfstudio.lantern.config import Get_lantern_config from nerfstudio.models.depth_nerfacto import DepthNerfactoModelConfig from nerfstudio.models.generfacto import GenerfactoModelConfig from nerfstudio.models.instant_ngp import InstantNGPModelConfig @@ -74,6 +80,9 @@ descriptions = { "nerfacto-HDR": "Nerfacto that take HDR as input, and output HDR field.", "lantern-nerfacto": "Nerfacto that train on two exposures.", "panoHDR-nerfacto": "Nerfacto that take HDR as input, and output HDR field.", + # "lantern-nerfacto": "Nerfacto that take HDR as input, and output HDR field.", + "hdr-nerfacto": "HDR-Nerfacto, adapted to the Nerfacto model.", + "hdr-nerfacto-wo-crf": "HDR-Nerfacto without CRF, adapted to the Nerfacto model.", "depth-nerfacto": "Nerfacto with depth supervision.", "volinga": "Real-time rendering model from Volinga. Directly exportable to NVOL format at https://volinga.ai/", "instant-ngp": "Implementation of Instant-NGP. Recommended real-time model for unbounded scenes.", @@ -125,6 +134,10 @@ method_configs["nerfacto"] = TrainerConfig( method_configs["lantern-nerfacto"] = Get_lantern_config() method_configs["panoHDR-nerfacto"] = Get_PanoHDR_Nerfacto_config() +method_configs["hdr-nerfacto"] = get_hdr_nerfacto_config(use_crf=True, clip_before_accumulation=False) +method_configs["hdr-nerfacto-wo-crf"] = get_hdr_nerfacto_config(use_crf=False, clip_before_accumulation=False) +method_configs["hdr-nerfacto-wo-crf-clip-bf-acc"] = get_hdr_nerfacto_config(use_crf=False, clip_before_accumulation=True) + method_configs["nerfacto-big"] = TrainerConfig( method_name="nerfacto", steps_per_eval_batch=500, diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index 8a693bd28..cb0434645 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -54,7 +54,6 @@ from nerfstudio.configs.dataparser_configs import AnnotatedDataParserUnion from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs from nerfstudio.data.dataparsers.blender_dataparser import BlenderDataParserConfig from nerfstudio.data.datasets.base_dataset import InputDataset -from nerfstudio.data.datasets.hdr_dataset import HDRInputDataset from nerfstudio.data.pixel_samplers import ( PixelSampler, diff --git a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py index 3c7f2f1f7..56dd4ca50 100644 --- a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py +++ b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py @@ -16,18 +16,17 @@ from __future__ import annotations import math +import os from dataclasses import dataclass, field from pathlib import Path from typing import Literal, Optional, Type -import os + os.environ["OPENCV_IO_ENABLE_OPENEXR"]="1" +import cv2 import numpy as np import torch -from PIL import Image -import cv2 - from nerfstudio.cameras import camera_utils from nerfstudio.cameras.cameras import CAMERA_MODEL_TO_TYPE, Cameras, CameraType diff --git a/nerfstudio/data/datasets/hdr_dataset.py b/nerfstudio/data/datasets/hdr_dataset.py deleted file mode 100644 index c6f1c8954..000000000 --- a/nerfstudio/data/datasets/hdr_dataset.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Dataset. -""" -from __future__ import annotations - -from copy import deepcopy -from pathlib import Path -from typing import Dict, List -import os -os.environ["OPENCV_IO_ENABLE_OPENEXR"]="1" - -import numpy as np -import numpy.typing as npt -import torch -from jaxtyping import Float -from PIL import Image -from torch import Tensor -from torch.utils.data import Dataset -import cv2 - - -from nerfstudio.cameras.cameras import Cameras -from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs -from nerfstudio.data.utils.data_utils import get_image_mask_tensor_from_path -from nerfstudio.data.datasets.base_dataset import InputDataset - -class HDRInputDataset(InputDataset): - """Dataset that returns HDR images, OpenEXR. - - Args: - dataparser_outputs: description of where and how to read input images. - scale_factor: The scaling factor for the dataparser outputs - """ - - # exclude_batch_keys_from_device: List[str] = ["image", "mask"] - # cameras: Cameras - - def __init__(self, dataparser_outputs: DataparserOutputs, scale_factor: float = 1.0): - super().__init__(dataparser_outputs, scale_factor) - - def __len__(self): - return len(self._dataparser_outputs.image_filenames) - - def get_numpy_image(self, image_idx: int) -> npt.NDArray[np.uint8]: - """Returns the image of shape (H, W, 3 or 4), with np.float32 values. - - Args: - image_idx: The image index in the dataset. - """ - image_filename = self._dataparser_outputs.image_filenames[image_idx] - hdr_image = cv2.imread(str(image_filename), cv2.IMREAD_UNCHANGED) - if self.scale_factor != 1.0: - # Here cv2 img shape is (height, width, BGR) - height, width = hdr_image.shape[:2] - newsize = (int(width * self.scale_factor), int(height * self.scale_factor)) - hdr_image = cv2.resize(hdr_image, newsize, interpolation = cv2.INTER_LINEAR) - hdr_image = cv2.cvtColor(hdr_image, cv2.COLOR_BGR2RGB) - hdr_image = hdr_image.astype("float32") # shape is (h, w) or (h, w, 3 or 4) - if len(hdr_image.shape) == 2: - hdr_image = hdr_image[:, :, None].repeat(3, axis=2) - assert len(hdr_image.shape) == 3 - assert hdr_image.dtype == np.float32 - assert hdr_image.shape[2] in [3, 4], f"Image shape of {hdr_image.shape} is in correct." - # dfdffdd - return hdr_image - - def get_image(self, image_idx: int) -> Float[Tensor, "image_height image_width num_channels"]: - """Returns a 3 channel image. - - Args: - image_idx: The image index in the dataset. - """ - # Compress HDR using: pixel = log(pixel + 1.) - - image = torch.from_numpy(self.get_numpy_image(image_idx).astype("float32")) - image = (image) ** (1 / 2.2) - image = torch.clip(image, 0, 1) - # image = torch.log(1. + image) - - - if self._dataparser_outputs.alpha_color is not None and image.shape[-1] == 4: - image = image[:, :, :3] * image[:, :, -1:] + self._dataparser_outputs.alpha_color * (1.0 - image[:, :, -1:]) - return image - - def get_data(self, image_idx: int) -> Dict: - """Returns the ImageDataset data as a dictionary. - - Args: - image_idx: The image index in the dataset. - """ - image = self.get_image(image_idx) - data = {"image_idx": image_idx, "image": image} - if self._dataparser_outputs.mask_filenames is not None: - mask_filepath = self._dataparser_outputs.mask_filenames[image_idx] - data["mask"] = get_image_mask_tensor_from_path(filepath=mask_filepath, scale_factor=self.scale_factor) - assert ( - data["mask"].shape[:2] == data["image"].shape[:2] - ), f"Mask and image have different shapes. Got {data['mask'].shape[:2]} and {data['image'].shape[:2]}" - metadata = self.get_metadata(data) - data.update(metadata) - return data - - def get_metadata(self, data: Dict) -> Dict: - """Method that can be used to process any additional metadata that may be part of the model inputs. - - Args: - image_idx: The image index in the dataset. - """ - del data - return {} - - def __getitem__(self, image_idx: int) -> Dict: - data = self.get_data(image_idx) - return data - - @property - def image_filenames(self) -> List[Path]: - """ - Returns image filenames for this dataset. - The order of filenames is the same as in the Cameras object for easy mapping. - """ - - return self._dataparser_outputs.image_filenames diff --git a/nerfstudio/data/pixel_samplers.py b/nerfstudio/data/pixel_samplers.py index 67c6c6b9b..6e3dd7285 100644 --- a/nerfstudio/data/pixel_samplers.py +++ b/nerfstudio/data/pixel_samplers.py @@ -120,8 +120,7 @@ class PixelSampler: del indice_this_disk return indices, nonzero_indices_whole - - + @profiler.time_function def sample_method( self, @@ -269,7 +268,6 @@ class PixelSampler: collated_batch = { key: value[chan, y, x] for key, value in batch.items() if key != "image_idx" and key != "exposure" and key != "image_filename" and value is not None } - assert collated_batch["image"].shape[0] == num_rays_per_batch # Needed to correct the random indices to their actual camera idx locations. diff --git a/nerfstudio/field_components/field_heads.py b/nerfstudio/field_components/field_heads.py index e67ebd557..d80b8bf7e 100644 --- a/nerfstudio/field_components/field_heads.py +++ b/nerfstudio/field_components/field_heads.py @@ -43,6 +43,13 @@ class FieldHeadNames(Enum): GRADIENT = "gradient" VALIDITY = 'validity' RGB_FAST = 'rgb_fast' + + RGB_HDR = "rgb_hdr" # HDR-Nerfacto` + RGB_LINEAR_CLIPPED = "rgb_linear_clipped" # HDR-Nerfacto + RGB_FAST = "rgb_fast" # HDR-Nerfacto + PRED_RGB_LDR = "pred_rgb_ldr" # HDR-Nerfacto + ZERO_RADIANCE_CRF = "zero_radiance_crf" # HDR-Nerfacto + class FieldHead(FieldComponent): """Base field output diff --git a/nerfstudio/hdr_nerfacto/__init__.py b/nerfstudio/hdr_nerfacto/__init__.py new file mode 100644 index 000000000..84952a816 --- /dev/null +++ b/nerfstudio/hdr_nerfacto/__init__.py @@ -0,0 +1 @@ +# init \ No newline at end of file diff --git a/nerfstudio/hdr_nerfacto/config.py b/nerfstudio/hdr_nerfacto/config.py new file mode 100644 index 000000000..7f3af16ba --- /dev/null +++ b/nerfstudio/hdr_nerfacto/config.py @@ -0,0 +1,59 @@ + +from nerfstudio.cameras.camera_optimizers import CameraOptimizerConfig +from nerfstudio.configs.base_config import ViewerConfig +from nerfstudio.data.dataparsers.nerfstudio_dataparser import NerfstudioDataParserConfig +from nerfstudio.engine.optimizers import AdamOptimizerConfig +from nerfstudio.engine.schedulers import ExponentialDecaySchedulerConfig +from nerfstudio.engine.trainer import TrainerConfig +from nerfstudio.hdr_nerfacto.hdr_nerf_model import HdrNerfactoModelConfig +from nerfstudio.lantern.datamanager import HDRNerfactoDataManagerConfig, HDRNerfactoWoCrfDataManagerConfig +from nerfstudio.pipelines.base_pipeline import VanillaPipelineConfig + + +def get_hdr_nerfacto_config(*, use_crf=True, clip_before_accumulation=False): + if use_crf: + method_name = "hdr-nerfacto" + assert not clip_before_accumulation + else: + if clip_before_accumulation: + method_name = "hdr-nerfacto-wo-crf-clip-bf-acc" + else: + method_name = "hdr-nerfacto-wo-crf" + + datamanager_config_class = HDRNerfactoDataManagerConfig if use_crf else HDRNerfactoWoCrfDataManagerConfig + + return TrainerConfig( + method_name=method_name, + steps_per_eval_batch=500, + steps_per_save=2000, + # max_num_iterations=1000, + max_num_iterations=60000, + mixed_precision=True, + pipeline=VanillaPipelineConfig( + datamanager=datamanager_config_class( + dataparser=NerfstudioDataParserConfig(), + train_num_rays_per_batch=4096, + eval_num_rays_per_batch=4096, + camera_optimizer=CameraOptimizerConfig( + mode="off" + ), + ), + model=HdrNerfactoModelConfig( + eval_num_rays_per_chunk=1 << 15, + use_crf=use_crf, + clip_before_accumulation=clip_before_accumulation, + ), + ), + optimizers={ + "proposal_networks": { + "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=0.0001, max_steps=200000), + }, + "fields": { + "optimizer": AdamOptimizerConfig(lr=1e-2, eps=1e-15), + "scheduler": ExponentialDecaySchedulerConfig(lr_final=0.0001, max_steps=200000), + } + }, + viewer=ViewerConfig(num_rays_per_chunk=1 << 15), + vis="viewer", + ) diff --git a/nerfstudio/hdr_nerfacto/hdr_nerf_field.py b/nerfstudio/hdr_nerfacto/hdr_nerf_field.py new file mode 100644 index 000000000..e5af2dff5 --- /dev/null +++ b/nerfstudio/hdr_nerfacto/hdr_nerf_field.py @@ -0,0 +1,355 @@ +# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Field for compound nerf model, adds scene contraction and image embeddings to instant ngp +""" + + +from typing import Dict, Literal, Optional, Tuple + +import torch +from torch import Tensor, nn + +from nerfstudio.cameras.rays import RaySamples +from nerfstudio.data.scene_box import SceneBox +from nerfstudio.field_components.activations import trunc_exp +from nerfstudio.field_components.embedding import Embedding +from nerfstudio.field_components.encodings import HashEncoding, NeRFEncoding, SHEncoding +from nerfstudio.field_components.field_heads import ( + FieldHeadNames, + PredNormalsFieldHead, + SemanticFieldHead, + TransientDensityFieldHead, + TransientRGBFieldHead, + UncertaintyFieldHead, +) +from nerfstudio.field_components.mlp import MLP +from nerfstudio.field_components.spatial_distortions import SpatialDistortion +from nerfstudio.fields.base_field import Field, get_normalized_directions + + +class HdrNerfactoField(Field): + """Compound Field that uses TCNN + + Args: + aabb: parameters of scene aabb bounds + num_images: number of images in the dataset + num_layers: number of hidden layers + hidden_dim: dimension of hidden layers + geo_feat_dim: output geo feat dimensions + num_levels: number of levels of the hashmap for the base mlp + base_res: base resolution of the hashmap for the base mlp + max_res: maximum resolution of the hashmap for the base mlp + log2_hashmap_size: size of the hashmap for the base mlp + num_layers_color: number of hidden layers for color network + num_layers_transient: number of hidden layers for transient network + features_per_level: number of features per level for the hashgrid + hidden_dim_color: dimension of hidden layers for color network + hidden_dim_transient: dimension of hidden layers for transient network + appearance_embedding_dim: dimension of appearance embedding + transient_embedding_dim: dimension of transient embedding + use_transient_embedding: whether to use transient embedding + use_semantics: whether to use semantic segmentation + num_semantic_classes: number of semantic classes + use_pred_normals: whether to use predicted normals + use_average_appearance_embedding: whether to use average appearance embedding or zeros for inference + spatial_distortion: spatial distortion to apply to the scene + """ + + aabb: Tensor + + def __init__( + self, + aabb: Tensor, + num_images: int, + num_layers: int = 2, + hidden_dim: int = 64, + geo_feat_dim: int = 15, + num_levels: int = 16, + base_res: int = 16, + max_res: int = 2048, + log2_hashmap_size: int = 19, + num_layers_color: int = 3, + num_layers_transient: int = 2, + features_per_level: int = 2, + hidden_dim_color: int = 64, + hidden_dim_transient: int = 64, + appearance_embedding_dim: int = 32, + transient_embedding_dim: int = 16, + use_transient_embedding: bool = False, + use_semantics: bool = False, + num_semantic_classes: int = 100, + pass_semantic_gradients: bool = False, + use_pred_normals: bool = False, + use_average_appearance_embedding: bool = False, + spatial_distortion: Optional[SpatialDistortion] = None, + implementation: Literal["tcnn", "torch"] = "tcnn", + use_crf: bool = True, + ) -> None: + super().__init__() + + self.register_buffer("aabb", aabb) + self.geo_feat_dim = geo_feat_dim + + self.register_buffer("max_res", torch.tensor(max_res)) + self.register_buffer("num_levels", torch.tensor(num_levels)) + self.register_buffer("log2_hashmap_size", torch.tensor(log2_hashmap_size)) + + self.spatial_distortion = spatial_distortion + self.num_images = num_images + # self.appearance_embedding_dim = appearance_embedding_dim + # self.embedding_appearance = Embedding(self.num_images, self.appearance_embedding_dim) + # self.use_average_appearance_embedding = use_average_appearance_embedding + self.use_transient_embedding = use_transient_embedding + self.use_semantics = use_semantics + self.use_pred_normals = use_pred_normals + self.pass_semantic_gradients = pass_semantic_gradients + self.base_res = base_res + + self.direction_encoding = SHEncoding( + levels=4, + implementation=implementation, + ) + + self.position_encoding = NeRFEncoding( + in_dim=3, num_frequencies=2, min_freq_exp=0, max_freq_exp=2 - 1, implementation=implementation + ) + + self.mlp_base_grid = HashEncoding( + num_levels=num_levels, + min_res=base_res, + max_res=max_res, + log2_hashmap_size=log2_hashmap_size, + features_per_level=features_per_level, + implementation=implementation, + ) + self.mlp_base_mlp = MLP( + in_dim=self.mlp_base_grid.get_out_dim(), + num_layers=num_layers, + layer_width=hidden_dim, + out_dim=1 + self.geo_feat_dim, + activation=nn.ReLU(), + out_activation=None, + implementation=implementation, + ) + self.mlp_base = torch.nn.Sequential(self.mlp_base_grid, self.mlp_base_mlp) + + # transients + if self.use_transient_embedding: + self.transient_embedding_dim = transient_embedding_dim + self.embedding_transient = Embedding(self.num_images, self.transient_embedding_dim) + self.mlp_transient = MLP( + in_dim=self.geo_feat_dim + self.transient_embedding_dim, + num_layers=num_layers_transient, + layer_width=hidden_dim_transient, + out_dim=hidden_dim_transient, + activation=nn.ReLU(), + out_activation=None, + implementation=implementation, + ) + self.field_head_transient_uncertainty = UncertaintyFieldHead(in_dim=self.mlp_transient.get_out_dim()) + self.field_head_transient_rgb = TransientRGBFieldHead(in_dim=self.mlp_transient.get_out_dim()) + self.field_head_transient_density = TransientDensityFieldHead(in_dim=self.mlp_transient.get_out_dim()) + + # semantics + if self.use_semantics: + self.mlp_semantics = MLP( + in_dim=self.geo_feat_dim, + num_layers=2, + layer_width=64, + out_dim=hidden_dim_transient, + activation=nn.ReLU(), + out_activation=None, + implementation=implementation, + ) + self.field_head_semantics = SemanticFieldHead( + in_dim=self.mlp_semantics.get_out_dim(), num_classes=num_semantic_classes + ) + + # predicted normals + if self.use_pred_normals: + self.mlp_pred_normals = MLP( + in_dim=self.geo_feat_dim + self.position_encoding.get_out_dim(), + num_layers=3, + layer_width=64, + out_dim=hidden_dim_transient, + activation=nn.ReLU(), + out_activation=None, + implementation=implementation, + ) + self.field_head_pred_normals = PredNormalsFieldHead(in_dim=self.mlp_pred_normals.get_out_dim()) + + self.mlp_head = MLP( + in_dim=self.direction_encoding.get_out_dim() + self.geo_feat_dim, + num_layers=num_layers_color, + layer_width=hidden_dim_color, + out_dim=3, + activation=nn.ReLU(), + out_activation=None, + implementation=implementation, + ) + + self.use_crf = use_crf + if use_crf: + # HDR NeRF, 1 MLP per channel + self.crf_mlps = nn.ModuleList([ + MLP(in_dim=1, + num_layers=2, + layer_width=128, + out_dim=1, + activation=nn.ReLU(), + out_activation=nn.Sigmoid(), + implementation=implementation, + ) for _ in range(3) + ]) + + + def get_density(self, ray_samples: RaySamples) -> Tuple[Tensor, Tensor]: + """Computes and returns the densities.""" + if self.spatial_distortion is not None: + positions = ray_samples.frustums.get_positions() + positions = self.spatial_distortion(positions) + positions = (positions + 2.0) / 4.0 + else: + positions = SceneBox.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) + # Make sure the tcnn gets inputs between 0 and 1. + selector = ((positions > 0.0) & (positions < 1.0)).all(dim=-1) + positions = positions * selector[..., None] + self._sample_locations = positions + if not self._sample_locations.requires_grad: + self._sample_locations.requires_grad = True + positions_flat = positions.view(-1, 3) + h = self.mlp_base(positions_flat).view(*ray_samples.frustums.shape, -1) + density_before_activation, base_mlp_out = torch.split(h, [1, self.geo_feat_dim], dim=-1) + self._density_before_activation = density_before_activation + + # Rectifying the density with an exponential is much more stable than a ReLU or + # softplus, because it enables high post-activation (float32) density outputs + # from smaller internal (float16) parameters. + density = trunc_exp(density_before_activation.to(positions)) + density = density * selector[..., None] + return density, base_mlp_out + + def get_outputs( + self, ray_samples: RaySamples, density_embedding: Optional[Tensor] = None + ) -> Dict[FieldHeadNames, Tensor]: + + if ray_samples.metadata is None or 'exposures' not in ray_samples.metadata.keys(): + exposures = torch.ones(ray_samples.frustums.directions.shape[:-1], device=ray_samples.frustums.directions.device)[..., None] + else: + exposures = ray_samples.metadata['exposures'].to(ray_samples.frustums.directions.device) + exposures = exposures.reshape(-1, 1).clone().detach() + + assert density_embedding is not None + outputs = {} + if ray_samples.camera_indices is None: + raise AttributeError("Camera indices are not provided.") + camera_indices = ray_samples.camera_indices.squeeze() + directions = get_normalized_directions(ray_samples.frustums.directions) + directions_flat = directions.view(-1, 3) + d = self.direction_encoding(directions_flat) + + outputs_shape = ray_samples.frustums.directions.shape[:-1] + + # # appearance + # if self.training: + # embedded_appearance = self.embedding_appearance(camera_indices) + # else: + # if self.use_average_appearance_embedding: + # embedded_appearance = torch.ones( + # (*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device + # ) * self.embedding_appearance.mean(dim=0) + # else: + # embedded_appearance = torch.zeros( + # (*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device + # ) + + # transients + if self.use_transient_embedding and self.training: + embedded_transient = self.embedding_transient(camera_indices) + transient_input = torch.cat( + [ + density_embedding.view(-1, self.geo_feat_dim), + embedded_transient.view(-1, self.transient_embedding_dim), + ], + dim=-1, + ) + x = self.mlp_transient(transient_input).view(*outputs_shape, -1).to(directions) + outputs[FieldHeadNames.UNCERTAINTY] = self.field_head_transient_uncertainty(x) + outputs[FieldHeadNames.TRANSIENT_RGB] = self.field_head_transient_rgb(x) + outputs[FieldHeadNames.TRANSIENT_DENSITY] = self.field_head_transient_density(x) + + # semantics + if self.use_semantics: + semantics_input = density_embedding.view(-1, self.geo_feat_dim) + if not self.pass_semantic_gradients: + semantics_input = semantics_input.detach() + + x = self.mlp_semantics(semantics_input).view(*outputs_shape, -1).to(directions) + outputs[FieldHeadNames.SEMANTICS] = self.field_head_semantics(x) + + # predicted normals + if self.use_pred_normals: + positions = ray_samples.frustums.get_positions() + + positions_flat = self.position_encoding(positions.view(-1, 3)) + pred_normals_inp = torch.cat([positions_flat, density_embedding.view(-1, self.geo_feat_dim)], dim=-1) + + x = self.mlp_pred_normals(pred_normals_inp).view(*outputs_shape, -1).to(directions) + outputs[FieldHeadNames.PRED_NORMALS] = self.field_head_pred_normals(x) + + h = torch.cat( + [ + d, + density_embedding.view(-1, self.geo_feat_dim), + # embedded_appearance.view(-1, self.appearance_embedding_dim), + ], + dim=-1, + ) + log_rgb_hdr = self.mlp_head(h) # .view(*outputs_shape, -1).to(directions) + + log_rgb_hdr_exposed = log_rgb_hdr + torch.log(exposures) + #print('exposure_min', exposures.min(), 'exposure_max', exposures.max()) + # TODO: validate this + rgb_hdr = torch.exp(log_rgb_hdr_exposed).view(*outputs_shape, -1).to(directions) + #print('rgb_hdr_min', rgb_hdr.min(), 'rgb_hdr_max', rgb_hdr.max()) + + outputs[FieldHeadNames.RGB_LINEAR_CLIPPED] = torch.clamp(rgb_hdr, 0, 1) + outputs[FieldHeadNames.RGB_HDR] = rgb_hdr + + if self.use_crf: + log_rgb_hdr_exposed_fast = log_rgb_hdr + torch.log(exposures * 0.004) # TODO: adaptive fast exposure + pred_rgb_fast = torch.cat([ + self.crf_mlps[channel](log_rgb_hdr_exposed_fast[..., channel:channel+1]) for channel in range(3) + ], dim=-1).view(*outputs_shape, -1).to(directions) + + pred_rgb_ldr = torch.cat([ + self.crf_mlps[channel](log_rgb_hdr_exposed[..., channel:channel+1]) for channel in range(3) + ], dim=-1) + pred_rgb_ldr = pred_rgb_ldr.view(*outputs_shape, -1).to(directions) + + zero_radiance_crf = torch.cat([ + self.crf_mlps[channel](torch.zeros((1, 1), requires_grad=False, device=log_rgb_hdr_exposed.device)) for channel in range(3) + ], dim=-1) + zero_radiance_crf = zero_radiance_crf.expand(pred_rgb_ldr.shape).to(directions) + + outputs.update({ + FieldHeadNames.RGB: pred_rgb_ldr, + FieldHeadNames.RGB_FAST: pred_rgb_fast, + FieldHeadNames.ZERO_RADIANCE_CRF: zero_radiance_crf, + }) + + return outputs + diff --git a/nerfstudio/hdr_nerfacto/hdr_nerf_model.py b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py new file mode 100644 index 000000000..5b14835b4 --- /dev/null +++ b/nerfstudio/hdr_nerfacto/hdr_nerf_model.py @@ -0,0 +1,462 @@ +# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +NeRF implementation that combines many recent advancements. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict, List, Literal, Tuple, Type + +import numpy as np +import torch +from torch.nn import Parameter +from torchmetrics.functional import structural_similarity_index_measure +from torchmetrics.image import PeakSignalNoiseRatio +from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity + +from nerfstudio.cameras.rays import RayBundle, RaySamples +from nerfstudio.engine.callbacks import ( + TrainingCallback, + TrainingCallbackAttributes, + TrainingCallbackLocation, +) +from nerfstudio.field_components.field_heads import FieldHeadNames +from nerfstudio.field_components.spatial_distortions import SceneContraction +from nerfstudio.fields.density_fields import HashMLPDensityField +from nerfstudio.hdr_nerfacto.hdr_nerf_field import HdrNerfactoField +from nerfstudio.lantern.renderer import RGBRenderer_HDR +from nerfstudio.model_components.losses import ( + MSELoss, + distortion_loss, + orientation_loss, + pred_normal_loss, + scale_gradients_by_distance_squared, +) +from nerfstudio.model_components.ray_samplers import ( + ProposalNetworkSampler, + UniformSampler, +) +from nerfstudio.model_components.renderers import ( + AccumulationRenderer, + DepthRenderer, + NormalsRenderer, + RGBRenderer, +) +from nerfstudio.model_components.scene_colliders import NearFarCollider +from nerfstudio.model_components.shaders import NormalsShader +from nerfstudio.models.base_model import Model, ModelConfig +from nerfstudio.utils import colormaps + + +@dataclass +class HdrNerfactoModelConfig(ModelConfig): + """Nerfacto Model Config""" + + _target: Type = field(default_factory=lambda: HdrNerfactoModel) + near_plane: float = 0.05 + """How far along the ray to start sampling.""" + far_plane: float = 1000.0 + """How far along the ray to stop sampling.""" + background_color: Literal["random", "last_sample", "black", "white"] = "last_sample" + """Whether to randomize the background color.""" + hidden_dim: int = 64 + """Dimension of hidden layers""" + hidden_dim_color: int = 64 + """Dimension of hidden layers for color network""" + hidden_dim_transient: int = 64 + """Dimension of hidden layers for transient network""" + num_levels: int = 16 + """Number of levels of the hashmap for the base mlp.""" + base_res: int = 16 + """Resolution of the base grid for the hashgrid.""" + max_res: int = 2048 + """Maximum resolution of the hashmap for the base mlp.""" + log2_hashmap_size: int = 19 + """Size of the hashmap for the base mlp""" + features_per_level: int = 2 + """How many hashgrid features per level""" + num_proposal_samples_per_ray: Tuple[int, ...] = (256, 96) + """Number of samples per ray for each proposal network.""" + num_nerf_samples_per_ray: int = 48 + """Number of samples per ray for the nerf network.""" + proposal_update_every: int = 5 + """Sample every n steps after the warmup""" + proposal_warmup: int = 5000 + """Scales n from 1 to proposal_update_every over this many steps""" + num_proposal_iterations: int = 2 + """Number of proposal network iterations.""" + use_same_proposal_network: bool = False + """Use the same proposal network. Otherwise use different ones.""" + proposal_net_args_list: List[Dict] = field( + default_factory=lambda: [ + {"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 128, "use_linear": False}, + {"hidden_dim": 16, "log2_hashmap_size": 17, "num_levels": 5, "max_res": 256, "use_linear": False}, + ] + ) + """Arguments for the proposal density fields.""" + proposal_initial_sampler: Literal["piecewise", "uniform"] = "piecewise" + """Initial sampler for the proposal network. Piecewise is preferred for unbounded scenes.""" + interlevel_loss_mult: float = 1.0 + """Proposal loss multiplier.""" + distortion_loss_mult: float = 0.002 + """Distortion loss multiplier.""" + orientation_loss_mult: float = 0.0001 + """Orientation loss multiplier on computed normals.""" + pred_normal_loss_mult: float = 0.001 + """Predicted normal loss multiplier.""" + use_proposal_weight_anneal: bool = True + """Whether to use proposal weight annealing.""" + use_average_appearance_embedding: bool = True + """Whether to use average appearance embedding or zeros for inference.""" + proposal_weights_anneal_slope: float = 10.0 + """Slope of the annealing function for the proposal weights.""" + proposal_weights_anneal_max_num_iters: int = 1000 + """Max num iterations for the annealing function.""" + use_single_jitter: bool = True + """Whether use single jitter or not for the proposal networks.""" + predict_normals: bool = False + """Whether to predict normals or not.""" + disable_scene_contraction: bool = False + """Whether to disable scene contraction or not.""" + use_gradient_scaling: bool = False + """Use gradient scaler where the gradients are lower for points closer to the camera.""" + implementation: Literal["tcnn", "torch"] = "tcnn" + """Which implementation to use for the model.""" + appearance_embed_dim: int = 32 + """Dimension of the appearance embedding.""" + + # HDR-Nerfacto + unit_exposure_loss_mult: float = 0.5 + unit_exposure_expected_value: float = 0.5 + + use_crf: bool = True + clip_before_accumulation: bool = False + + +class HdrNerfactoModel(Model): + """Nerfacto model + + Args: + config: Nerfacto configuration to instantiate model + """ + + config: HdrNerfactoModelConfig + + def populate_modules(self): + """Set the fields and modules.""" + super().populate_modules() + + if self.config.disable_scene_contraction: + scene_contraction = None + else: + scene_contraction = SceneContraction(order=float("inf")) + + # Fields + self.field = HdrNerfactoField( + self.scene_box.aabb, + hidden_dim=self.config.hidden_dim, + num_levels=self.config.num_levels, + max_res=self.config.max_res, + base_res=self.config.base_res, + features_per_level=self.config.features_per_level, + log2_hashmap_size=self.config.log2_hashmap_size, + hidden_dim_color=self.config.hidden_dim_color, + hidden_dim_transient=self.config.hidden_dim_transient, + spatial_distortion=scene_contraction, + num_images=self.num_train_data, + use_pred_normals=self.config.predict_normals, + use_average_appearance_embedding=self.config.use_average_appearance_embedding, + appearance_embedding_dim=self.config.appearance_embed_dim, + implementation=self.config.implementation, + use_crf=self.config.use_crf, + ) + + self.density_fns = [] + num_prop_nets = self.config.num_proposal_iterations + # Build the proposal network(s) + self.proposal_networks = torch.nn.ModuleList() + if self.config.use_same_proposal_network: + assert len(self.config.proposal_net_args_list) == 1, "Only one proposal network is allowed." + prop_net_args = self.config.proposal_net_args_list[0] + network = HashMLPDensityField( + self.scene_box.aabb, + spatial_distortion=scene_contraction, + **prop_net_args, + implementation=self.config.implementation, + ) + self.proposal_networks.append(network) + self.density_fns.extend([network.density_fn for _ in range(num_prop_nets)]) + else: + for i in range(num_prop_nets): + prop_net_args = self.config.proposal_net_args_list[min(i, len(self.config.proposal_net_args_list) - 1)] + network = HashMLPDensityField( + self.scene_box.aabb, + spatial_distortion=scene_contraction, + **prop_net_args, + implementation=self.config.implementation, + ) + self.proposal_networks.append(network) + self.density_fns.extend([network.density_fn for network in self.proposal_networks]) + + # Samplers + def update_schedule(step): + return np.clip( + np.interp(step, [0, self.config.proposal_warmup], [0, self.config.proposal_update_every]), + 1, + self.config.proposal_update_every, + ) + + # Change proposal network initial sampler if uniform + initial_sampler = None # None is for piecewise as default (see ProposalNetworkSampler) + if self.config.proposal_initial_sampler == "uniform": + initial_sampler = UniformSampler(single_jitter=self.config.use_single_jitter) + + self.proposal_sampler = ProposalNetworkSampler( + num_nerf_samples_per_ray=self.config.num_nerf_samples_per_ray, + num_proposal_samples_per_ray=self.config.num_proposal_samples_per_ray, + num_proposal_network_iterations=self.config.num_proposal_iterations, + single_jitter=self.config.use_single_jitter, + update_sched=update_schedule, + initial_sampler=initial_sampler, + ) + + # Collider + self.collider = NearFarCollider(near_plane=self.config.near_plane, far_plane=self.config.far_plane) + + # renderers + self.renderer_rgb = RGBRenderer(background_color=self.config.background_color) + self.renderer_rgb_hdr = RGBRenderer_HDR(background_color=self.config.background_color) + self.renderer_accumulation = AccumulationRenderer() + self.renderer_depth = DepthRenderer() + self.renderer_normals = NormalsRenderer() + + # shaders + self.normals_shader = NormalsShader() + + # losses + self.rgb_loss = MSELoss() + self.unit_exposure_loss = MSELoss() + + # metrics + self.psnr = PeakSignalNoiseRatio(data_range=1.0) + self.ssim = structural_similarity_index_measure + self.lpips = LearnedPerceptualImagePatchSimilarity(normalize=True) + + def get_param_groups(self) -> Dict[str, List[Parameter]]: + param_groups = {} + param_groups["proposal_networks"] = list(self.proposal_networks.parameters()) + param_groups["fields"] = list(self.field.parameters()) + return param_groups + + def get_training_callbacks( + self, training_callback_attributes: TrainingCallbackAttributes + ) -> List[TrainingCallback]: + callbacks = [] + if self.config.use_proposal_weight_anneal: + # anneal the weights of the proposal network before doing PDF sampling + N = self.config.proposal_weights_anneal_max_num_iters + + def set_anneal(step): + # https://arxiv.org/pdf/2111.12077.pdf eq. 18 + train_frac = np.clip(step / N, 0, 1) + + def bias(x, b): + return b * x / ((b - 1) * x + 1) + + anneal = bias(train_frac, self.config.proposal_weights_anneal_slope) + self.proposal_sampler.set_anneal(anneal) + + callbacks.append( + TrainingCallback( + where_to_run=[TrainingCallbackLocation.BEFORE_TRAIN_ITERATION], + update_every_num_iters=1, + func=set_anneal, + ) + ) + callbacks.append( + TrainingCallback( + where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION], + update_every_num_iters=1, + func=self.proposal_sampler.step_cb, + ) + ) + return callbacks + + def get_outputs(self, ray_bundle: RayBundle): + ray_samples: RaySamples + ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns) + field_outputs = self.field.forward(ray_samples, compute_normals=self.config.predict_normals) + if self.config.use_gradient_scaling: + field_outputs = scale_gradients_by_distance_squared(field_outputs, ray_samples) + + weights = ray_samples.get_weights(field_outputs[FieldHeadNames.DENSITY]) + weights_list.append(weights) + ray_samples_list.append(ray_samples) + + # HDR-Nerfacto + rgb_hdr = self.renderer_rgb_hdr( + rgb=field_outputs[FieldHeadNames.RGB_HDR], weights=weights + ) + if self.config.clip_before_accumulation: + rgb_linear_clipped = self.renderer_rgb( + rgb=field_outputs[FieldHeadNames.RGB_LINEAR_CLIPPED], weights=weights + ) + else: + # clipping only after accumulation (in the renderer) + rgb_linear_clipped = self.renderer_rgb( + rgb=field_outputs[FieldHeadNames.RGB_HDR], weights=weights + ) + + depth = self.renderer_depth(weights=weights, ray_samples=ray_samples) + accumulation = self.renderer_accumulation(weights=weights) + + outputs = { + "accumulation": accumulation, + "depth": depth, + "rgb_hdr": rgb_hdr, + "rgb_linear_clipped": rgb_linear_clipped, + } + if self.config.use_crf: + rgb = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB], weights=weights) + rgb_fast = self.renderer_rgb(rgb=field_outputs[FieldHeadNames.RGB_FAST], weights=weights) + outputs["rgb"] = rgb + outputs["rgb_fast"] = rgb_fast + outputs['zero_radiance_crf'] = field_outputs[FieldHeadNames.ZERO_RADIANCE_CRF] + + if self.config.predict_normals: + normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights) + pred_normals = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights) + outputs["normals"] = self.normals_shader(normals) + outputs["pred_normals"] = self.normals_shader(pred_normals) + # These use a lot of GPU memory, so we avoid storing them for eval. + if self.training: + outputs["weights_list"] = weights_list + outputs["ray_samples_list"] = ray_samples_list + + if self.training and self.config.predict_normals: + outputs["rendered_orientation_loss"] = orientation_loss( + weights.detach(), field_outputs[FieldHeadNames.NORMALS], ray_bundle.directions + ) + + outputs["rendered_pred_normal_loss"] = pred_normal_loss( + weights.detach(), + field_outputs[FieldHeadNames.NORMALS].detach(), + field_outputs[FieldHeadNames.PRED_NORMALS], + ) + + for i in range(self.config.num_proposal_iterations): + outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i]) + + return outputs + + def get_metrics_dict(self, outputs, batch): + metrics_dict = {} + gt_rgb = batch["image"].to(self.device) # RGB or RGBA image + gt_rgb = self.renderer_rgb.blend_background(gt_rgb) # Blend if RGBA + if self.config.use_crf: + predicted_rgb = outputs["rgb"] + else: + predicted_rgb = outputs["rgb_hdr"] + metrics_dict["psnr"] = self.psnr(predicted_rgb, gt_rgb) + + if self.training: + metrics_dict["distortion"] = distortion_loss(outputs["weights_list"], outputs["ray_samples_list"]) + return metrics_dict + + def get_loss_dict(self, outputs, batch, metrics_dict=None): + loss_dict = {} + image = batch["image"].to(self.device) + + if self.config.use_crf: + pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation( + pred_image=outputs["rgb"], + pred_accumulation=outputs["accumulation"], + gt_image=image, + ) + loss_dict["unit_exposure_loss"] = self.config.unit_exposure_loss_mult * self.unit_exposure_loss( + outputs["zero_radiance_crf"], torch.ones_like(outputs["zero_radiance_crf"], requires_grad=False) * self.config.unit_exposure_expected_value + ) + else: + pred_rgb, gt_rgb = self.renderer_rgb.blend_background_for_loss_computation( + pred_image=outputs["rgb_linear_clipped"], + pred_accumulation=outputs["accumulation"], + gt_image=image, + ) + + loss_dict["rgb_loss"] = self.rgb_loss(gt_rgb, pred_rgb) + + # TODO: check that + # if self.training: + # loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss( + # outputs["weights_list"], outputs["ray_samples_list"] + # ) + # assert metrics_dict is not None and "distortion" in metrics_dict + # loss_dict["distortion_loss"] = self.config.distortion_loss_mult * metrics_dict["distortion"] + # if self.config.predict_normals: + # # orientation loss for computed normals + # loss_dict["orientation_loss"] = self.config.orientation_loss_mult * torch.mean( + # outputs["rendered_orientation_loss"] + # ) +# + # # ground truth supervision for normals + # loss_dict["pred_normal_loss"] = self.config.pred_normal_loss_mult * torch.mean( + # outputs["rendered_pred_normal_loss"] + # ) + return loss_dict + + def get_image_metrics_and_images( + self, outputs: Dict[str, torch.Tensor], batch: Dict[str, torch.Tensor] + ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: + gt_rgb = batch["image"].to(self.device) + if self.config.use_crf: + predicted_rgb = outputs["rgb"] + else: + predicted_rgb = outputs["rgb_linear_clipped"] + gt_rgb = self.renderer_rgb.blend_background(gt_rgb) + acc = colormaps.apply_colormap(outputs["accumulation"]) + depth = colormaps.apply_depth_colormap( + outputs["depth"], + accumulation=outputs["accumulation"], + ) + + combined_rgb = predicted_rgb + combined_acc = torch.cat([acc], dim=1) + combined_depth = torch.cat([depth], dim=1) + + # Switch images from [H, W, C] to [1, C, H, W] for metrics computations + gt_rgb = torch.moveaxis(gt_rgb, -1, 0)[None, ...] + predicted_rgb = torch.moveaxis(predicted_rgb, -1, 0)[None, ...] + + psnr = self.psnr(gt_rgb, predicted_rgb) + ssim = self.ssim(gt_rgb, predicted_rgb) + lpips = self.lpips(torch.clamp(gt_rgb, 0, 1), torch.clamp(predicted_rgb, 0, 1)) + + # all of these metrics will be logged as scalars + metrics_dict = {"psnr": float(psnr.item()), "ssim": float(ssim)} # type: ignore + metrics_dict["lpips"] = float(lpips) + + images_dict = {"img": combined_rgb, "accumulation": combined_acc, "depth": combined_depth} + + for i in range(self.config.num_proposal_iterations): + key = f"prop_depth_{i}" + prop_depth_i = colormaps.apply_depth_colormap( + outputs[key], + accumulation=outputs["accumulation"], + ) + images_dict[key] = prop_depth_i + + return metrics_dict, images_dict diff --git a/nerfstudio/lantern/datamanager.py b/nerfstudio/lantern/datamanager.py index 90d8885aa..b9e2a5d3f 100644 --- a/nerfstudio/lantern/datamanager.py +++ b/nerfstudio/lantern/datamanager.py @@ -3,16 +3,34 @@ Lantern HDR DataManager """ from dataclasses import dataclass, field -from typing import Dict, Literal, Tuple, Type, Union +from typing import Type -import torch - -from nerfstudio.cameras.rays import RayBundle from nerfstudio.data.datamanagers.base_datamanager import ( - VanillaDataManager, VanillaDataManagerConfig) -from nerfstudio.lantern.dataset import HDRInputDataset + VanillaDataManager, + VanillaDataManagerConfig, +) +from nerfstudio.lantern.dataset import ( + HDRInputDataset, + HDRNerfactoInputDataset, + HDRNerfactoWoCrfInputDataset, +) +@dataclass +class HDRNerfactoDataManagerConfig(VanillaDataManagerConfig): + """A HDR data manager, based on VanillaDataManager""" + + _target: Type = field(default_factory=lambda: VanillaDataManager[HDRNerfactoInputDataset]) + """Target class to instantiate.""" + +@dataclass +class HDRNerfactoWoCrfDataManagerConfig(VanillaDataManagerConfig): + """A HDR data manager, based on VanillaDataManager""" + + _target: Type = field(default_factory=lambda: VanillaDataManager[HDRNerfactoWoCrfInputDataset]) + """Target class to instantiate.""" + + @dataclass class HDRVanillaDataManagerConfig(VanillaDataManagerConfig): """A HDR data manager, based on VanillaDataManager""" diff --git a/nerfstudio/lantern/dataset.py b/nerfstudio/lantern/dataset.py index a0b5a7c80..052ad558a 100644 --- a/nerfstudio/lantern/dataset.py +++ b/nerfstudio/lantern/dataset.py @@ -18,7 +18,6 @@ HDR Dataset of .exr format from __future__ import annotations import os -from copy import deepcopy from pathlib import Path from typing import Dict, List @@ -29,16 +28,246 @@ import numpy as np import numpy.typing as npt import torch from jaxtyping import Float -from PIL import Image from torch import Tensor -from torch.utils.data import Dataset -from nerfstudio.cameras.cameras import Cameras from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs from nerfstudio.data.datasets.base_dataset import InputDataset from nerfstudio.data.utils.data_utils import get_image_mask_tensor_from_path +class HDRNerfactoInputDataset(InputDataset): + """Dataset that returns HDR images, OpenEXR. + + Args: + dataparser_outputs: description of where and how to read input images. + scale_factor: The scaling factor for the dataparser outputs + """ + + # exclude_batch_keys_from_device: List[str] = ["image", "mask"] + # cameras: Cameras + + def __init__(self, dataparser_outputs: DataparserOutputs, scale_factor: float = 1.0): + super().__init__(dataparser_outputs, scale_factor) + + def __len__(self): + return len(self._dataparser_outputs.image_filenames) + + def get_numpy_image(self, image_idx: int) -> npt.NDArray[np.uint8]: + """Returns the image of shape (H, W, 3 or 4), with np.float32 values. + + Args: + image_idx: The image index in the dataset. + """ + image_filename = str(self._dataparser_outputs.image_filenames[image_idx]).replace('_linear.exr', '.png') + #print('image_filename', image_filename) + hdr_image = cv2.imread(str(image_filename)) + # print('hdr_image', hdr_image.shape, hdr_image.dtype) + # print('hdr_image', hdr_image.max(), hdr_image.min()) + # print('hdr_image', hdr_image[0,0,:]) + + if self.scale_factor != 1.0: + # Here cv2 img shape is (height, width, BGR) + height, width = hdr_image.shape[:2] + newsize = (int(width * self.scale_factor), int(height * self.scale_factor)) + hdr_image = cv2.resize(hdr_image, newsize, interpolation = cv2.INTER_LINEAR) + hdr_image = cv2.cvtColor(hdr_image, cv2.COLOR_BGR2RGB) + hdr_image = hdr_image.astype("float32") / 255 # shape is (h, w) or (h, w, 3 or 4) + if len(hdr_image.shape) == 2: + hdr_image = hdr_image[:, :, None].repeat(3, axis=2) + assert len(hdr_image.shape) == 3 + assert hdr_image.dtype == np.float32 + assert hdr_image.shape[2] in [3, 4], f"Image shape of {hdr_image.shape} is in correct." + return hdr_image + + def get_image(self, image_idx: int) -> Float[Tensor, "image_height image_width num_channels"]: + """Returns a 3 channel image. + + Args: + image_idx: The image index in the dataset. + """ + image = torch.from_numpy(self.get_numpy_image(image_idx).astype("float32")) + #print('a', image.max()) + # image = image * self._dataparser_outputs.exposures[image_idx] + # print('b', image.max()) + + # if self._dataparser_outputs.alpha_color is not None and image.shape[-1] == 4: + # image = image[:, :, :3] * image[:, :, -1:] + self._dataparser_outputs.alpha_color * (1.0 - image[:, :, -1:]) + return image + + def get_data(self, image_idx: int) -> Dict: + """Returns the ImageDataset data as a dictionary. + + Args: + image_idx: The image index in the dataset. + """ + image = self.get_image(image_idx) + data = {"image_idx": image_idx, "image": image} + + # TODO: this doesnt work for training + image_filename = self._dataparser_outputs.image_filenames[image_idx] + data["image_filename"] = str(image_filename.name) + + if self._dataparser_outputs.mask_filenames is not None: + mask_filepath = self._dataparser_outputs.mask_filenames[image_idx] + data["mask"] = get_image_mask_tensor_from_path(filepath=mask_filepath, scale_factor=self.scale_factor) + assert ( + data["mask"].shape[:2] == data["image"].shape[:2] + ), f"Mask and image have different shapes. Got {data['mask'].shape[:2]} and {data['image'].shape[:2]}" + + if self._dataparser_outputs.saturation_mask_filenames is not None: + saturation_mask_filepath = self._dataparser_outputs.saturation_mask_filenames[image_idx] + data["saturation_mask"] = get_image_mask_tensor_from_path(filepath=saturation_mask_filepath, scale_factor=self.scale_factor) + assert ( + data["saturation_mask"].shape[:2] == data["image"].shape[:2] + ), f"Saturation mask and image have different shapes. Got {data['saturation_mask'].shape[:2]} and {data['image'].shape[:2]}" + + if self._dataparser_outputs.exposures is not None: + exposure_value = self._dataparser_outputs.exposures[image_idx] + data["exposure"] = exposure_value + metadata = self.get_metadata(data) + data.update(metadata) + return data + + def get_metadata(self, data: Dict) -> Dict: + """Method that can be used to process any additional metadata that may be part of the model inputs. + + Args: + image_idx: The image index in the dataset. + """ + del data + return {} + + def __getitem__(self, image_idx: int) -> Dict: + data = self.get_data(image_idx) + return data + + @property + def image_filenames(self) -> List[Path]: + """ + Returns image filenames for this dataset. + The order of filenames is the same as in the Cameras object for easy mapping. + """ + + return self._dataparser_outputs.image_filenames + + +class HDRNerfactoWoCrfInputDataset(InputDataset): + """Dataset that returns HDR images, OpenEXR. + + Args: + dataparser_outputs: description of where and how to read input images. + scale_factor: The scaling factor for the dataparser outputs + """ + + # exclude_batch_keys_from_device: List[str] = ["image", "mask"] + # cameras: Cameras + + def __init__(self, dataparser_outputs: DataparserOutputs, scale_factor: float = 1.0): + super().__init__(dataparser_outputs, scale_factor) + + def __len__(self): + return len(self._dataparser_outputs.image_filenames) + + def get_numpy_image(self, image_idx: int) -> npt.NDArray[np.uint8]: + """Returns the image of shape (H, W, 3 or 4), with np.float32 values. + + Args: + image_idx: The image index in the dataset. + """ + image_filename = self._dataparser_outputs.image_filenames[image_idx] + #print('image_filename', image_filename) + hdr_image = cv2.imread(str(image_filename), cv2.IMREAD_UNCHANGED) + # print('hdr_image', hdr_image.shape, hdr_image.dtype) + # print('hdr_image', hdr_image.max(), hdr_image.min()) + # print('hdr_image', hdr_image[0,0,:]) + + if self.scale_factor != 1.0: + # Here cv2 img shape is (height, width, BGR) + height, width = hdr_image.shape[:2] + newsize = (int(width * self.scale_factor), int(height * self.scale_factor)) + hdr_image = cv2.resize(hdr_image, newsize, interpolation = cv2.INTER_LINEAR) + hdr_image = cv2.cvtColor(hdr_image, cv2.COLOR_BGR2RGB) + hdr_image = hdr_image.astype("float32") # shape is (h, w) or (h, w, 3 or 4) + if len(hdr_image.shape) == 2: + hdr_image = hdr_image[:, :, None].repeat(3, axis=2) + assert len(hdr_image.shape) == 3 + assert hdr_image.dtype == np.float32 + assert hdr_image.shape[2] in [3, 4], f"Image shape of {hdr_image.shape} is in correct." + return hdr_image + + def get_image(self, image_idx: int) -> Float[Tensor, "image_height image_width num_channels"]: + """Returns a 3 channel image. + + Args: + image_idx: The image index in the dataset. + """ + image = torch.from_numpy(self.get_numpy_image(image_idx).astype("float32")) + #print('a', image.max()) + # image = image * self._dataparser_outputs.exposures[image_idx] + # print('b', image.max()) + + # if self._dataparser_outputs.alpha_color is not None and image.shape[-1] == 4: + # image = image[:, :, :3] * image[:, :, -1:] + self._dataparser_outputs.alpha_color * (1.0 - image[:, :, -1:]) + return image + + def get_data(self, image_idx: int) -> Dict: + """Returns the ImageDataset data as a dictionary. + + Args: + image_idx: The image index in the dataset. + """ + image = self.get_image(image_idx) + data = {"image_idx": image_idx, "image": image} + + # TODO: this doesnt work for training + image_filename = self._dataparser_outputs.image_filenames[image_idx] + data["image_filename"] = str(image_filename.name) + + if self._dataparser_outputs.mask_filenames is not None: + mask_filepath = self._dataparser_outputs.mask_filenames[image_idx] + data["mask"] = get_image_mask_tensor_from_path(filepath=mask_filepath, scale_factor=self.scale_factor) + assert ( + data["mask"].shape[:2] == data["image"].shape[:2] + ), f"Mask and image have different shapes. Got {data['mask'].shape[:2]} and {data['image'].shape[:2]}" + + if self._dataparser_outputs.saturation_mask_filenames is not None: + saturation_mask_filepath = self._dataparser_outputs.saturation_mask_filenames[image_idx] + data["saturation_mask"] = get_image_mask_tensor_from_path(filepath=saturation_mask_filepath, scale_factor=self.scale_factor) + assert ( + data["saturation_mask"].shape[:2] == data["image"].shape[:2] + ), f"Saturation mask and image have different shapes. Got {data['saturation_mask'].shape[:2]} and {data['image'].shape[:2]}" + + if self._dataparser_outputs.exposures is not None: + exposure_value = self._dataparser_outputs.exposures[image_idx] + data["exposure"] = exposure_value + metadata = self.get_metadata(data) + data.update(metadata) + return data + + def get_metadata(self, data: Dict) -> Dict: + """Method that can be used to process any additional metadata that may be part of the model inputs. + + Args: + image_idx: The image index in the dataset. + """ + del data + return {} + + def __getitem__(self, image_idx: int) -> Dict: + data = self.get_data(image_idx) + return data + + @property + def image_filenames(self) -> List[Path]: + """ + Returns image filenames for this dataset. + The order of filenames is the same as in the Cameras object for easy mapping. + """ + + return self._dataparser_outputs.image_filenames + + + class HDRInputDataset(InputDataset): """Dataset that returns HDR images, OpenEXR. diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py index b4d36ccc9..429021422 100644 --- a/nerfstudio/models/nerfacto.py +++ b/nerfstudio/models/nerfacto.py @@ -29,7 +29,11 @@ from torchmetrics.image import PeakSignalNoiseRatio from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity from nerfstudio.cameras.rays import RayBundle, RaySamples -from nerfstudio.engine.callbacks import TrainingCallback, TrainingCallbackAttributes, TrainingCallbackLocation +from nerfstudio.engine.callbacks import ( + TrainingCallback, + TrainingCallbackAttributes, + TrainingCallbackLocation, +) from nerfstudio.field_components.field_heads import FieldHeadNames from nerfstudio.field_components.spatial_distortions import SceneContraction from nerfstudio.fields.density_fields import HashMLPDensityField @@ -42,8 +46,16 @@ from nerfstudio.model_components.losses import ( pred_normal_loss, scale_gradients_by_distance_squared, ) -from nerfstudio.model_components.ray_samplers import ProposalNetworkSampler, UniformSampler -from nerfstudio.model_components.renderers import AccumulationRenderer, DepthRenderer, NormalsRenderer, RGBRenderer +from nerfstudio.model_components.ray_samplers import ( + ProposalNetworkSampler, + UniformSampler, +) +from nerfstudio.model_components.renderers import ( + AccumulationRenderer, + DepthRenderer, + NormalsRenderer, + RGBRenderer, +) from nerfstudio.model_components.scene_colliders import NearFarCollider from nerfstudio.model_components.shaders import NormalsShader from nerfstudio.models.base_model import Model, ModelConfig diff --git a/nerfstudio/scripts/render.py b/nerfstudio/scripts/render.py index 0bc1a4659..16c84fec0 100644 --- a/nerfstudio/scripts/render.py +++ b/nerfstudio/scripts/render.py @@ -177,6 +177,10 @@ def _render_trajectory_video( # img_uncompress /= u # output_image = img_uncompress # output_image = (output_image.cpu().numpy()) + elif rendered_output_name == "rgb_fast" or rendered_output_name == "rgb" or rendered_output_name == "rgb_hdr": + output_image = ( + output_image.cpu().numpy() + ) elif is_validity: mask_f = torch.clip(outputs["validity_f"], 0, 1) # mask_f = torch.ones(mask_f.shape).to(mask_f.get_device()) - mask_f diff --git a/nerfstudio/utils/colormaps.py b/nerfstudio/utils/colormaps.py index 07d6b1415..9510e11c9 100644 --- a/nerfstudio/utils/colormaps.py +++ b/nerfstudio/utils/colormaps.py @@ -43,6 +43,11 @@ class ColormapOptions: """ Whether to invert the output colormap """ exposure_scale: float = 0 """Scale the brightness prior to tonemapping by 2 ^ Exposure""" + use_mu_law: bool = False + """Use mu-law de-compression for HDR images""" + use_gamma: bool = False + """Use gamma for HDR images""" + def apply_colormap( image: Float[Tensor, "*bs channels"], diff --git a/nerfstudio/viewer/server/control_panel.py b/nerfstudio/viewer/server/control_panel.py index b55885614..d1766e10c 100644 --- a/nerfstudio/viewer/server/control_panel.py +++ b/nerfstudio/viewer/server/control_panel.py @@ -79,8 +79,8 @@ class ControlPanel: self._exposure = ViewerSlider( "Exposure scale", default_value=0.0, - min_value=-10.0, - max_value=10.0, + min_value=-20.0, + max_value=20.0, step=0.1, cb_hook=rerender_cb, hint="Scale the brightness prior to tonemapping by 2 ^ Exposure" @@ -224,7 +224,7 @@ class ControlPanel: Sets elements to be hidden or not based on the current state of the control panel """ self._colormap.set_disabled(self.output_render == "rgb") - self._exposure.set_disabled(self.output_render != "rgb") + self._exposure.set_disabled(self.output_render not in ["rgb", "rgb_hdr"]) for e in self._elements_by_tag["colormap"]: e.set_hidden(self.output_render == "rgb") for e in self._elements_by_tag["split_colormap"]: diff --git a/video_rendering_code/hdr_blender.py b/video_rendering_code/hdr_blender.py new file mode 100644 index 000000000..d6e0da3a7 --- /dev/null +++ b/video_rendering_code/hdr_blender.py @@ -0,0 +1,58 @@ +""" +Script to render IBL renders +Usage: blender -P hdr_blender.py -b +""" +import os +import random +from math import radians +from os import listdir +from os.path import isfile, join + +import bpy +import cycles +from mathutils import Euler, Matrix, Vector + + +import sys +argv = sys.argv +argv = argv[argv.index("--") + 1:] # get all args after "--" + +inp_dirs = [argv[0]] +out_dirs = [argv[1]] + +pairs = list(zip(inp_dirs,out_dirs)) +scn = bpy.context.scene +scn.render.engine = 'CYCLES' +scn.render.image_settings.file_format = 'OPEN_EXR' +scn.world.use_nodes = True +# Enabf_le GPU rendering. +# bpy.context.preferences.addons['cycles'].preferences.compute_device_type = 'CUDA' +bpy.context.preferences.addons['cycles'].preferences.get_devices() +for device in bpy.context.preferences.addons['cycles'].preferences.devices: + if "TITAN RX" == device.name: + device.use = True + else: + device.use = False +# select world node tree +wd = scn.world +nt = bpy.data.worlds[wd.name].node_tree + +for pair in pairs: + inp, out = pair + inp = inp + os.makedirs(out,exist_ok=True) + S = bpy.context.scene + envFiles = [f for f in listdir(inp) if isfile(join(inp, f)) and f.endswith('.exr')] + i = 0 + for env_map in envFiles: + env_file = join(inp, env_map) + bpy.ops.image.open(filepath=env_file, directory=inp, files=[ + {"name": env_map, "name": env_map}], show_multiview=False) + output_name = env_map.split('.')[0] +'.exr' + + backNode = nt.nodes['Environment Texture'] + backNode.image = bpy.data.images.load(env_file) + + bpy.context.scene.render.filepath = os.path.join(out, output_name) + print("Output",os.path.join(out, output_name)) + bpy.ops.render.render(write_still=True)