diff --git a/lantern_scripts/combine.py b/lantern_scripts/combine.py index 0ae17ed2a..7c2035113 100755 --- a/lantern_scripts/combine.py +++ b/lantern_scripts/combine.py @@ -7,7 +7,7 @@ import argparse import json -FAST_EXPOSURE_CUTOFF = 0.2 # good for real data (0.1 for synthetic) +FAST_EXPOSURE_CUTOFF = 0.1 # good for real data (0.1 for synthetic) WELL_EXPOSURE_CUTOFF = 0.98 # good for real data (0.9 for synthetic) @@ -94,8 +94,9 @@ if __name__ == "__main__": # fast_mask = cv2.imread(well_mask_addr.replace('/well_mask/', '/fast_mask/')) # fast_mask = fast_mask.astype(np.float32) / 255.0 - final_output = weights_well * (hdr_well_image / WELL_EXPOSURE) + weights_fast * (hdr_fast_image / FAST_EXPOSURE) - # final_output = weights_well * (hdr_well_image / WELL_EXPOSURE) + (hdr_fast_image / FAST_EXPOSURE) + # final_output = weights_well * (hdr_well_image / WELL_EXPOSURE) + weights_fast * (hdr_fast_image / FAST_EXPOSURE) + # final_output = final_output / (weights_well+weights_well) + final_output = weights_well * (hdr_well_image / WELL_EXPOSURE) + (hdr_fast_image / FAST_EXPOSURE) # final_output = weights_well * (hdr_well_image / WELL_EXPOSURE) output_addr = well_addr.replace(args.well_dir, args.out_dir) diff --git a/lantern_scripts/combine_well_at_fast.py b/lantern_scripts/combine_well_at_fast.py new file mode 100755 index 000000000..943d3c028 --- /dev/null +++ b/lantern_scripts/combine_well_at_fast.py @@ -0,0 +1,110 @@ +import os +import numpy as np +import cv2 +from pathlib import Path +from tqdm import tqdm +import argparse +import json + + +FAST_EXPOSURE_CUTOFF = 0.01 # good for real data (0.1 for synthetic) +WELL_EXPOSURE_CUTOFF = 0.98 # good for real data (0.9 for synthetic) + + +def apply_correction(coefs, img, CORRECTION_CURVE_TYPE) -> np.ndarray: + # check that img is in range 0-255 + assert img.dtype == np.uint8 + assert CORRECTION_CURVE_TYPE in ['gamma'] + img = ((img.astype(np.float32) / 255.0) ** np.array(coefs).T) # * 255.0 + # img = ((img.astype(np.float32) / 255.0) ** coefs[None, None, :]) * 255.0 + return img + +def cut_weighting_function(pixels, exposures): + weights = np.zeros(pixels.shape, dtype=np.float32) + + if exposures == WELL_EXPOSURE: + weights = np.clip(-20.0 * (pixels - WELL_EXPOSURE_CUTOFF) + 1, 0.0, 1.0) + else: + weights = np.clip(100.0 * (pixels - FAST_EXPOSURE_CUTOFF), 0.0, 1.0) + + return weights + + +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--well_dir', type=str, default='/mnt/workspace/lantern/nerfstudio-HDR/well_at_fast/') + parser.add_argument('--fast_dir', type=str, default='/mnt/data/nerfstudio_ds/real_data/meeting_room_openSFM/meeting_room_ns/images/') + parser.add_argument('--out_dir', type=str, default='./output_well_at_fast/') + parser.add_argument('--do_linearization', action='store_true') + parser.add_argument('--experiment_location', type=str, default='/mnt/data/nerfstudio_ds/real_data/meeting_room_openSFM') + args = parser.parse_args() + + experiment_setup = args.experiment_location + '/capture_settings.json' + assert os.path.exists(experiment_setup), "!!!! Cannot find the files for exposure setup. !!!" + with open(experiment_setup) as f: + data = json.load(f) + WELL_EXPOSURE = 1.0 + FAST_EXPOSURE = data["right"]["shutter_speed"] / data["left"]["shutter_speed"] + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + well_addr = args.well_dir + + with open(f'linearize-ricoh-z10-wb-3500.json') as f: + camera_data = json.load(f) + CORRECTION_CURVE_TYPE = 'gamma' + + well_images = [] + if args.do_linearization: + for path in Path(well_addr).rglob('*-img.png'): + well_images.append(path) + else: + for path in Path(well_addr).rglob('*.exr'): + well_images.append(path) + + for well_addr in tqdm(well_images): + + well_addr = str(well_addr) + + print("well_addr: ", well_addr) + + hdr_well_image = cv2.imread(well_addr, cv2.IMREAD_UNCHANGED) + fast_addr = well_addr.replace(args.well_dir, args.fast_dir) + fast_addr = fast_addr.replace("-img.png", ".png") + hdr_fast_image = cv2.imread(fast_addr, cv2.IMREAD_UNCHANGED) + + + + # hdr_well_image = np.uint8(hdr_well_image * 255.0) + # hdr_fast_image = np.uint8(hdr_fast_image * 255.0) + + if args.do_linearization: + hdr_well_image = apply_correction((camera_data["b"], camera_data["g"], camera_data["r"]), np.array(hdr_well_image), CORRECTION_CURVE_TYPE) + hdr_fast_image = apply_correction((camera_data["b"], camera_data["g"], camera_data["r"]), np.array(hdr_fast_image), CORRECTION_CURVE_TYPE) + + + # well_mask_addr = well_addr.replace('/well_expo/', '/well_mask/') + # well_mask_addr = well_mask_addr.replace('.exr', '.png') + # well_mask = cv2.imread(well_mask_addr) + # well_mask = well_mask.astype(np.float32) / 255.0 + # well_mask[well_mask > 0.1] = 1.0 + + weights_well = cut_weighting_function(hdr_well_image, WELL_EXPOSURE) + weights_fast = cut_weighting_function(hdr_fast_image, FAST_EXPOSURE) + + # fast_mask = cv2.imread(well_mask_addr.replace('/well_mask/', '/fast_mask/')) + # fast_mask = fast_mask.astype(np.float32) / 255.0 + + # final_output = weights_well * (hdr_well_image / WELL_EXPOSURE) + weights_fast * (hdr_fast_image / FAST_EXPOSURE) + final_output = weights_well * (hdr_well_image / WELL_EXPOSURE) + (hdr_fast_image / FAST_EXPOSURE) + # final_output = weights_well * (hdr_well_image / WELL_EXPOSURE) + + output_addr = well_addr.replace(args.well_dir, args.out_dir) + if args.do_linearization: + output_addr = output_addr.replace('-img.png', '.exr') + cv2.imwrite(output_addr, np.float32(final_output) ) + # import pdb; pdb.set_trace() \ No newline at end of file diff --git a/lantern_scripts/ldr_res.py b/lantern_scripts/ldr_res.py old mode 100644 new mode 100755 diff --git a/lantern_scripts/linearize_jpgs.py b/lantern_scripts/linearize_jpgs.py old mode 100644 new mode 100755 index 04927e916..797395a48 --- a/lantern_scripts/linearize_jpgs.py +++ b/lantern_scripts/linearize_jpgs.py @@ -25,6 +25,9 @@ if __name__ == "__main__": args = parser.parse_args() well_addr = args.data_dir + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + with open(f'./linearize-ricoh-z10-wb-3500.json') as f: camera_data = json.load(f) CORRECTION_CURVE_TYPE = 'gamma' diff --git a/lantern_scripts/masking_panos.py b/lantern_scripts/masking_panos.py new file mode 100755 index 000000000..ca9d4c2aa --- /dev/null +++ b/lantern_scripts/masking_panos.py @@ -0,0 +1,38 @@ +import argparse +import numpy as np +import os +from tqdm import tqdm +from PIL import Image +from pathlib import Path +import PIL.ExifTags +import json +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" +import cv2 +import warnings + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--data_dir', type=str, default='/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/GT/pano/') + parser.add_argument('--mask_dir', type=str, default='/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/GT/mask/') + parser.add_argument('--out_dir', type=str, default='/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/GT/masked_pano/') + args = parser.parse_args() + + test_images = [] + for path in Path(args.data_dir).rglob('*.exr'): + test_images.append(path) + + for pano_addr in tqdm(test_images): + img = cv2.imread(str(pano_addr), cv2.IMREAD_UNCHANGED) + + mask_addr = str(pano_addr).replace(args.data_dir, args.mask_dir) + mask_addr = mask_addr.replace('.exr', '.png') + mask = cv2.imread(mask_addr, cv2.IMREAD_UNCHANGED) + mask = np.float32(mask) / 255. + + # import pdb; pdb.set_trace() + masked_img = np.expand_dims(mask, axis=-1) * img + # import pdb; pdb.set_trace() + out_addr = str(pano_addr).replace(args.data_dir, args.out_dir) + # import pdb; pdb.set_trace() + cv2.imwrite(out_addr, np.float32(masked_img)) \ No newline at end of file diff --git a/lantern_scripts/mp4_maker.py b/lantern_scripts/mp4_maker.py new file mode 100755 index 000000000..1403c6c0e --- /dev/null +++ b/lantern_scripts/mp4_maker.py @@ -0,0 +1,38 @@ +import cv2 +import numpy as np +import argparse + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Make a GIF from input images') + parser.add_argument("--input_Addr", type=str, help='input folder') + # parser.add_argument("--input_pano_Addr", type=str, help='input folder') + args = parser.parse_args() + + images = [] + + from os import listdir + from os.path import isfile, join + + img_files = [join(args.input_Addr, f) for f in listdir(args.input_Addr) if isfile(join(args.input_Addr, f))] + # pano_files = [join(args.input_pano_Addr, f) for f in listdir(args.input_pano_Addr) if isfile(join(args.input_pano_Addr, f))] + img_files = sorted(img_files) + + for i in range(len(img_files)): + img_add = img_files[i] + # pano_add = pano_files[i] + print(img_add) + # print(pano_add) + img = cv2.imread(img_add) + height, width, layers = img.shape + size = (width, height) + # pano = cv2.imread(pano_add) + # small_pano = cv2.resize(pano, (0, 0), fx=0.3, fy=0.3) + # img[0:small_pano.shape[0], 0:small_pano.shape[1]] = small_pano + images.append(img) + + out = cv2.VideoWriter('saussan_ldr_light.mp4', cv2.VideoWriter_fourcc(*'FMP4'), 10, size) + # out = cv2.VideoWriter('my_apartement_render.mp4', cv2.VideoWriter_fourcc('X','V','I','D'), 15, size) + + for i in range(len(images)): + out.write(images[i]) + out.release() \ No newline at end of file diff --git a/lantern_scripts/rectify_pano.py b/lantern_scripts/rectify_pano.py old mode 100644 new mode 100755 diff --git a/lantern_scripts/render_all.py b/lantern_scripts/render_all.py old mode 100644 new mode 100755 diff --git a/lantern_scripts/res_table_1.py b/lantern_scripts/res_table_1.py old mode 100644 new mode 100755 diff --git a/lantern_scripts/un_gamma.py b/lantern_scripts/un_gamma.py old mode 100644 new mode 100755 index 9253534ef..bc2310475 --- a/lantern_scripts/un_gamma.py +++ b/lantern_scripts/un_gamma.py @@ -17,6 +17,9 @@ if __name__ == "__main__": parser.add_argument('--out_dir', type=str, default='/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/Henry_Gagnon_red_and_blue/PanoHDR-NeRF/pano/') args = parser.parse_args() + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + test_images = [] for path in Path(args.data_dir).rglob('*.exr'): test_images.append(path) diff --git a/lantern_scripts/un_mu_law.py b/lantern_scripts/un_mu_law.py index d907f75ca..1e3e28dfe 100755 --- a/lantern_scripts/un_mu_law.py +++ b/lantern_scripts/un_mu_law.py @@ -15,14 +15,19 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--data_dir', type=str, default='/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/linearization_then_nerf/fast_compressed_mu_law/') parser.add_argument('--out_dir', type=str, default='//Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/linearization_then_nerf/fast_expo/') + parser.add_argument('--is_jpg', action='store_true') args = parser.parse_args() if not os.path.exists(args.out_dir): os.makedirs(args.out_dir) test_images = [] - for path in Path(args.data_dir).rglob('*.exr'): - test_images.append(path) + if args.is_jpg: + for path in Path(args.data_dir).rglob('*.jpg'): + test_images.append(path) + else: + for path in Path(args.data_dir).rglob('*.exr'): + test_images.append(path) for pano_addr in tqdm(test_images): pano_addr = str(pano_addr) @@ -39,4 +44,7 @@ if __name__ == "__main__": data = np.float32(data) data = cv2.cvtColor(data, cv2.COLOR_RGB2BGR) - cv2.imwrite(out_dir, data) \ No newline at end of file + if args.is_jpg: + cv2.imwrite(out_dir, data*255) + else: + cv2.imwrite(out_dir, data) \ No newline at end of file diff --git a/linearization_then_nerf_evaluation.sh b/linearization_then_nerf_evaluation.sh new file mode 100755 index 000000000..96d71c3c4 --- /dev/null +++ b/linearization_then_nerf_evaluation.sh @@ -0,0 +1,31 @@ +#!/bin/bash +echo "Evaluation for linearization then nerf pipeline" +root_dir=/mnt/data/nerfstudio_ds/real_data/Henry_Gagnon_blue_et_rouge +dataset_dir=$root_dir/Henry_Gagnon_blue_et_rouge_ns/ + +opensfm_dir=$root_dir/Henry_Gagnon_blue_et_rouge_openSFM +opensfm_file=$opensfm_dir/reconstruction.json + +checkpoint_dir=outputs/Henry_Gagnon_blue_et_rouge_ns/lantern-nerfacto/2024-05-16_030223/ +output_dir=renders/Henry_Gagnon_blue_et_rouge_ns-nerfacto/nerf_then_linearization/2024-05-16_030223_output + +#echo "Creating the test set" +#ns-process-data lantern-GT-HDR --data $dataset_dir --output-dir $dataset_dir --metadata $opensfm_file --checkpoint $checkpoint_dir + +#echo "Rendering for the well exposed" +#ns-render camera-path --load-config $checkpoint_dir/config.yml --camera-path-filename $dataset_dir/GT_transforms.json --output-path $output_dir/well-expo/ --output-format images --image-format exr + +#echo "Rendering for the fast exposure" +#ns-render camera-path --load-config $checkpoint_dir/config.yml --camera-path-filename $dataset_dir/GT_transforms.json --output-path $output_dir/fast-mu-law_compressed/ --output-format images --image-format exr --rendered_output_names rgb_fast + +echo "un-mu-law the fast exposure" +#conda init bash +conda activate everlight_new +python lantern_scripts/un_mu_law.py --data_dir $output_dir/fast-mu-law_compressed/ --out_dir $output_dir/fast-expo/ --is_jpg + +echo "Combine two exposures" +python lantern_scripts/combine.py --well_dir $output_dir/well-expo/ --fast_dir $output_dir/fast-expo/ --out_dir $output_dir/pano/ --experiment_location $root_dir + +conda activate nerfstudio-HDR + +blender --background lantern_scripts/scene_new.blend -P lantern_scripts/hdr_blender.py -- $output_dir/pano/ $output_dir/render/ \ No newline at end of file diff --git a/nerf_ablation_train_and_test_lab_downstairs.sh b/nerf_ablation_train_and_test_lab_downstairs.sh new file mode 100755 index 000000000..17bd00b6e --- /dev/null +++ b/nerf_ablation_train_and_test_lab_downstairs.sh @@ -0,0 +1,42 @@ +#!/bin/bash +echo "Evaluation for linearization then nerf pipeline" + +dataset_name=meeting_room_ns +method_name=lantern-nerfacto +root_dir=/mnt/data/nerfstudio_ds/real_data/meeting_room_openSFM +dataset_dir=$root_dir/$dataset_name/ +opensfm_dir=$root_dir/meeting_room_openSFM +opensfm_file=$opensfm_dir/reconstruction.json + +echo "Step one of training" +ns-train lantern-nerfacto --data $dataset_dir --viewer.websocket-port 8008 --pipeline.datamanager.train-num-images-to-sample-from 1800 \ +--pipeline.model.lantern_steps 1 --pipeline.datamanager.pixel-sampler.lantern_steps 1 --pipeline.model.apply_mu_law False \ +--pipeline.datamanager.camera-optimizer.mode off --viewer.quit-on-train-completion True --output-dir outputs --timestamp ablation_step_1 + +echo "Step two of training" +ns-train lantern-nerfacto --data $dataset_dir --viewer.websocket-port 8008 --pipeline.datamanager.train-num-images-to-sample-from 1800 \ +--pipeline.model.lantern_steps 2 --pipeline.datamanager.pixel-sampler.lantern_steps 2 --pipeline.model.apply_mu_law False \ +--pipeline.datamanager.camera-optimizer.mode off --viewer.quit-on-train-completion True --load-dir outputs/$dataset_name/lantern-nerfacto/ablation_step_1/nerfstudio_models \ +--output-dir outputs --timestamp ablation_step_2 + +# checkpoint_dir=outputs/$dataset_name/lantern-nerfacto/ablation_step_2/ +# output_dir=renders/$dataset_name/$method_name/ablation_linearization_then_nerf + +# echo "Creating the test set" +# ns-process-data lantern-GT-HDR --data $dataset_dir --output-dir $dataset_dir --metadata $opensfm_file --checkpoint $checkpoint_dir + +# echo "Rendering for the well exposed" +# ns-render camera-path --load-config $checkpoint_dir/config.yml --camera-path-filename $dataset_dir/GT_transforms.json --output-path $output_dir/well-expo/ --output-format images + +# echo "Rendering for the fast exposure" +# ns-render camera-path --load-config $checkpoint_dir/config.yml --camera-path-filename $dataset_dir/GT_transforms.json --output-path $output_dir/fast-expo/ --output-format images --rendered_output_names rgb_fast + +# #conda init bash +# conda activate everlight_new + +# echo "Combine two exposures" +# python lantern_scripts/combine.py --well_dir $output_dir/well-expo/ --fast_dir $output_dir/fast-expo/ --out_dir $output_dir/pano/ --experiment_location $root_dir + +# conda activate nerfstudio-HDR + +# blender --background lantern_scripts/scene_new.blend -P lantern_scripts/hdr_blender.py -- $output_dir/pano/ $output_dir/render/ \ No newline at end of file diff --git a/nerf_then_linearization_train_and_test.sh b/nerf_then_linearization_train_and_test.sh new file mode 100755 index 000000000..b68972b50 --- /dev/null +++ b/nerf_then_linearization_train_and_test.sh @@ -0,0 +1,42 @@ +#!/bin/bash +echo "Evaluation for linearization then nerf pipeline" + +dataset_name=meeting_room_ns +method_name=lantern-nerfacto +root_dir=/mnt/data/nerfstudio_ds/real_data/meeting_room_openSFM +dataset_dir=$root_dir/$dataset_name/ +opensfm_dir=$root_dir/meeting_room_openSFM +opensfm_file=$opensfm_dir/reconstruction.json + +echo "Step one of training" +#ns-train lantern-nerfacto --data $dataset_dir --viewer.websocket-port 8008 --pipeline.datamanager.train-num-images-to-sample-from 1800 \ +#--pipeline.model.lantern_steps 1 --pipeline.datamanager.pixel-sampler.lantern_steps 1 --pipeline.model.apply_mu_law False \ +#--pipeline.datamanager.camera-optimizer.mode off --viewer.quit-on-train-completion True --output-dir outputs --timestamp step_1 + +echo "Step two of training" +#ns-train lantern-nerfacto --data $dataset_dir --viewer.websocket-port 8008 --pipeline.datamanager.train-num-images-to-sample-from 1800 \ +#--pipeline.model.lantern_steps 2 --pipeline.datamanager.pixel-sampler.lantern_steps 2 --pipeline.model.apply_mu_law False \ +#--pipeline.datamanager.camera-optimizer.mode off --viewer.quit-on-train-completion True --load-dir outputs/$dataset_name/lantern-nerfacto/step_1/nerfstudio_models \ +#--output-dir outputs --timestamp step_2 + +checkpoint_dir=outputs/$dataset_name/lantern-nerfacto/step_2/ +output_dir=renders/$dataset_name/$method_name/ours_results + +echo "Creating the test set" +ns-process-data lantern-GT-HDR --data $dataset_dir --output-dir $dataset_dir --metadata $opensfm_file --checkpoint $checkpoint_dir + +echo "Rendering for the well exposed" +ns-render camera-path --load-config $checkpoint_dir/config.yml --camera-path-filename $dataset_dir/GT_transforms.json --output-path $output_dir/well-expo/ --output-format images + +echo "Rendering for the fast exposure" +ns-render camera-path --load-config $checkpoint_dir/config.yml --camera-path-filename $dataset_dir/GT_transforms.json --output-path $output_dir/fast-expo/ --output-format images --rendered_output_names rgb_fast + +#conda init bash +conda activate everlight_new + +echo "Combine two exposures" +python lantern_scripts/combine.py --well_dir $output_dir/well-expo/ --fast_dir $output_dir/fast-expo/ --out_dir $output_dir/pano/ --do_linearization --experiment_location $root_dir + +conda activate nerfstudio-HDR + +blender --background lantern_scripts/scene_new.blend -P lantern_scripts/hdr_blender.py -- $output_dir/pano/ $output_dir/render/ \ No newline at end of file diff --git a/nerf_training_lab_downstairs.sh b/nerf_training_lab_downstairs.sh new file mode 100755 index 000000000..833de59f9 --- /dev/null +++ b/nerf_training_lab_downstairs.sh @@ -0,0 +1,25 @@ +#!/bin/bash +echo "Evaluation for linearization then nerf pipeline" + +dataset_name=lab_downstairs_ns +method_name=lantern-nerfacto +root_dir=/mnt/data/nerfstudio_ds/real_data/lab_downstairs_data +dataset_dir=$root_dir/$dataset_name/ +opensfm_dir=$root_dir/lab_downstairs_openSFM +opensfm_file=$opensfm_dir/reconstruction.json + +echo "Step one of training" +ns-train lantern-nerfacto --data $dataset_dir --viewer.websocket-port 8008 --pipeline.datamanager.train-num-images-to-sample-from 1800 \ +--pipeline.model.lantern_steps 1 --pipeline.datamanager.pixel-sampler.lantern_steps 1 \ +--pipeline.datamanager.camera-optimizer.mode off --viewer.quit-on-train-completion True --output-dir outputs --timestamp step_1 + +echo "Step two of training" +#ns-train lantern-nerfacto --data $dataset_dir --viewer.websocket-port 8008 --pipeline.datamanager.train-num-images-to-sample-from 1800 \ +#--pipeline.model.lantern_steps 2 --pipeline.datamanager.pixel-sampler.lantern_steps 2 --pipeline.model.apply_mu_law False \ +#--pipeline.datamanager.camera-optimizer.mode off --viewer.quit-on-train-completion True --load-dir outputs/$dataset_name/lantern-nerfacto/step_1/nerfstudio_models \ +#--output-dir outputs --timestamp step_2 + + ns-train lantern-nerfacto --data $dataset_dir --viewer.websocket-port 8008 --pipeline.datamanager.train-num-images-to-sample-from 1800 \ + --pipeline.model.lantern_steps 2 --pipeline.datamanager.pixel-sampler.lantern_steps 2 \ + --pipeline.datamanager.camera-optimizer.mode off --load-dir outputs/$dataset_name/lantern-nerfacto/step_1/nerfstudio_models \ + --output-dir outputs --timestamp step_2 \ No newline at end of file diff --git a/nerf_training_lvsn.sh b/nerf_training_lvsn.sh new file mode 100755 index 000000000..5e4f3c1e1 --- /dev/null +++ b/nerf_training_lvsn.sh @@ -0,0 +1,23 @@ +#!/bin/bash +echo "Evaluation for linearization then nerf pipeline" + +dataset_name=LVSN_LAB_OpenSFM +method_name=lantern-nerfacto +root_dir=/mnt/data/nerfstudio_ds/real_data/lab_ground_floor_wb_3500/openSFM_full_lvsn +dataset_dir=$root_dir/$dataset_name/ + +echo "Step one of training" +# ns-train lantern-nerfacto --data $dataset_dir --viewer.websocket-port 8008 --pipeline.datamanager.train-num-images-to-sample-from 1800 \ +# --pipeline.model.lantern_steps 1 --pipeline.datamanager.pixel-sampler.lantern_steps 1 --pipeline.model.apply_mu_law False \ +# --pipeline.datamanager.camera-optimizer.mode off --viewer.quit-on-train-completion True --output-dir outputs --timestamp step_1 + +echo "Step two of training" +ns-train lantern-nerfacto --data $dataset_dir --viewer.websocket-port 8008 --pipeline.datamanager.train-num-images-to-sample-from 1800 \ +--pipeline.model.lantern_steps 2 --pipeline.datamanager.pixel-sampler.lantern_steps 2 --pipeline.model.apply_mu_law False \ +--pipeline.datamanager.camera-optimizer.mode off --viewer.quit-on-train-completion True --load-dir outputs/$dataset_name/lantern-nerfacto/step_1/nerfstudio_models \ +--output-dir outputs --timestamp step_2 + +# ns-train lantern-nerfacto --data $dataset_dir --viewer.websocket-port 8008 --pipeline.datamanager.train-num-images-to-sample-from 1800 \ +# --pipeline.model.lantern_steps 2 --pipeline.datamanager.pixel-sampler.lantern_steps 2 \ +# --pipeline.datamanager.camera-optimizer.mode off --load-dir outputs/$dataset_name/lantern-nerfacto/step_1/nerfstudio_models \ +# --output-dir outputs --timestamp step_2 \ No newline at end of file diff --git a/nerf_training_my_apartment.sh b/nerf_training_my_apartment.sh new file mode 100755 index 000000000..de5854d53 --- /dev/null +++ b/nerf_training_my_apartment.sh @@ -0,0 +1,25 @@ +#!/bin/bash +echo "Evaluation for linearization then nerf pipeline" + +dataset_name=my_apartment_ns +method_name=lantern-nerfacto +root_dir=/mnt/data/nerfstudio_ds/real_data/my_apartment_corrected +dataset_dir=$root_dir/$dataset_name/ +opensfm_dir=$root_dir/my_apartment_openSFM +opensfm_file=$opensfm_dir/reconstruction.json + +echo "Step one of training" +#ns-train lantern-nerfacto --data $dataset_dir --viewer.websocket-port 8008 --pipeline.datamanager.train-num-images-to-sample-from 1800 \ +#--pipeline.model.lantern_steps 1 --pipeline.datamanager.pixel-sampler.lantern_steps 1 --pipeline.model.apply_mu_law False \ +#--pipeline.datamanager.camera-optimizer.mode off --viewer.quit-on-train-completion True --output-dir outputs --timestamp step_1 + +echo "Step two of training" +#ns-train lantern-nerfacto --data $dataset_dir --viewer.websocket-port 8008 --pipeline.datamanager.train-num-images-to-sample-from 1800 \ +#--pipeline.model.lantern_steps 2 --pipeline.datamanager.pixel-sampler.lantern_steps 2 --pipeline.model.apply_mu_law False \ +#--pipeline.datamanager.camera-optimizer.mode off --viewer.quit-on-train-completion True --load-dir outputs/$dataset_name/lantern-nerfacto/step_1/nerfstudio_models \ +#--output-dir outputs --timestamp step_2 + +#ns-train lantern-nerfacto --data $dataset_dir --viewer.websocket-port 8008 --pipeline.datamanager.train-num-images-to-sample-from 1800 \ +#--pipeline.model.lantern_steps 2 --pipeline.datamanager.pixel-sampler.lantern_steps 2 \ +#--pipeline.datamanager.camera-optimizer.mode off --load-dir outputs/$dataset_name/lantern-nerfacto/step_1/nerfstudio_models \ +#--output-dir outputs --timestamp step_2 \ No newline at end of file diff --git a/nerfstudio/data/pixel_samplers.py b/nerfstudio/data/pixel_samplers.py index 2e1d3fed4..67c6c6b9b 100644 --- a/nerfstudio/data/pixel_samplers.py +++ b/nerfstudio/data/pixel_samplers.py @@ -392,24 +392,24 @@ class PixelSampler: self.fast_image_batch, int(0.0 * self.num_rays_per_batch), keep_full_image=self.config.keep_full_image, is_fast_expo=True, only_light_sources=True ) else: - # well_pixel_batch = self.collate_image_dataset_batch( - # self.well_image_batch, int(0.2 * self.num_rays_per_batch), keep_full_image=self.config.keep_full_image - # ) - # fast_pixel_batch = self.collate_image_dataset_batch( - # self.fast_image_batch, int(0.3 * self.num_rays_per_batch), keep_full_image=self.config.keep_full_image, is_fast_expo=True - # ) - # light_pixel_batch = self.collate_image_dataset_batch( - # self.fast_image_batch, int(0.5 * self.num_rays_per_batch), keep_full_image=self.config.keep_full_image, is_fast_expo=True, only_light_sources=True - # ) well_pixel_batch = self.collate_image_dataset_batch( - self.well_image_batch, int(0.0 * self.num_rays_per_batch), keep_full_image=self.config.keep_full_image + self.well_image_batch, int(0.2 * self.num_rays_per_batch), keep_full_image=self.config.keep_full_image ) fast_pixel_batch = self.collate_image_dataset_batch( self.fast_image_batch, int(0.3 * self.num_rays_per_batch), keep_full_image=self.config.keep_full_image, is_fast_expo=True ) light_pixel_batch = self.collate_image_dataset_batch( - self.fast_image_batch, int(0.7 * self.num_rays_per_batch), keep_full_image=self.config.keep_full_image, is_fast_expo=True, only_light_sources=True + self.fast_image_batch, int(0.5 * self.num_rays_per_batch), keep_full_image=self.config.keep_full_image, is_fast_expo=True, only_light_sources=True ) + # well_pixel_batch = self.collate_image_dataset_batch( + # self.well_image_batch, int(0.0 * self.num_rays_per_batch), keep_full_image=self.config.keep_full_image + # ) + # fast_pixel_batch = self.collate_image_dataset_batch( + # self.fast_image_batch, int(0.3 * self.num_rays_per_batch), keep_full_image=self.config.keep_full_image, is_fast_expo=True + # ) + # light_pixel_batch = self.collate_image_dataset_batch( + # self.fast_image_batch, int(0.7 * self.num_rays_per_batch), keep_full_image=self.config.keep_full_image, is_fast_expo=True, only_light_sources=True + # ) pixel_batch = well_pixel_batch #.copy() for key, value in fast_pixel_batch.items(): diff --git a/nerfstudio/lantern/config.py b/nerfstudio/lantern/config.py index 0aacdecdd..e09036291 100644 --- a/nerfstudio/lantern/config.py +++ b/nerfstudio/lantern/config.py @@ -30,12 +30,12 @@ def Get_lantern_config(): dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=8192, eval_num_rays_per_batch=8192, - camera_optimizer=CameraOptimizerConfig( - mode="SO3xR3", - optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2), - scheduler=ExponentialDecaySchedulerConfig(lr_final=6e-6, max_steps=200000), - ), - # camera_optimizer=None, + # camera_optimizer=CameraOptimizerConfig( + # mode="SO3xR3", + # optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2), + # scheduler=ExponentialDecaySchedulerConfig(lr_final=6e-6, max_steps=200000), + # ), + camera_optimizer=None, ), model=LanternModelConfig(eval_num_rays_per_chunk=1 << 15), ), diff --git a/nerfstudio/lantern/field.py b/nerfstudio/lantern/field.py index aee80d9dd..d9cb8d9ea 100644 --- a/nerfstudio/lantern/field.py +++ b/nerfstudio/lantern/field.py @@ -130,11 +130,11 @@ class LanternNerfactoField(NerfactoField): def get_density(self, ray_samples: RaySamples) -> Tuple[Tensor, Tensor]: """Computes and returns the densities.""" - if self.second_step: - with torch.no_grad(): - density, base_mlp_out = super().get_density(ray_samples) - else: - density, base_mlp_out = super().get_density(ray_samples) + # if self.second_step: + # with torch.no_grad(): + # density, base_mlp_out = super().get_density(ray_samples) + # else: + density, base_mlp_out = super().get_density(ray_samples) return density, base_mlp_out diff --git a/nerfstudio/utils/colormaps.py b/nerfstudio/utils/colormaps.py index 81ddad3ee..07d6b1415 100644 --- a/nerfstudio/utils/colormaps.py +++ b/nerfstudio/utils/colormaps.py @@ -66,11 +66,11 @@ def apply_colormap( # default for rgb images if image.shape[-1] == 3: # if(colormap_options.exposure_scale != 0 or image.max() > 1): - # if colormap_options.colormap == 'inverse-mu': - # u = 5000. - # img_uncompress = torch.exp(image * torch.log(torch.tensor(u+1.))) - 1. - # img_uncompress /= u - # image = img_uncompress + if colormap_options.colormap == 'inverse-mu': + u = 5000. + img_uncompress = torch.exp(image * torch.log(torch.tensor(u+1.))) - 1. + img_uncompress /= u + image = img_uncompress img_gamma_22 = image img_gamma_22 *= 2.**(colormap_options.exposure_scale) # img_gamma_22 = torch.pow(img_gamma_22, 1./2.2) diff --git a/pano_hdr_nerf_evaluation.sh b/pano_hdr_nerf_evaluation.sh new file mode 100755 index 000000000..53f227471 --- /dev/null +++ b/pano_hdr_nerf_evaluation.sh @@ -0,0 +1,25 @@ +#!/bin/bash +echo "Evaluation for linearization then nerf pipeline" + +opensfm_dir=/mnt/data/nerfstudio_ds/real_data/my_apartment_corrected/my_apartment_openSFM +opensfm_file=$opensfm_dir/reconstruction.json + +dataset_dir=/mnt/data/nerfstudio_ds/real_data/my_apartment_corrected/pano_hdr_nerf/pano_hdr_nerf_my_apt/ + +checkpoint_dir=outputs/pano_hdr_nerf_my_apt/panoHDR-nerfacto/2024-05-18_160257/ +output_dir=/mnt/workspace/lantern/nerfstudio-HDR/renders/pano_hdr_nerf_my_apt + + +echo "Creating the test set" +ns-process-data lantern-GT-HDR --data $dataset_dir --output-dir $dataset_dir --metadata $opensfm_file --checkpoint $checkpoint_dir + +echo "Rendering for the well exposed" +ns-render camera-path --load-config $checkpoint_dir/config.yml --camera-path-filename $dataset_dir/GT_transforms.json --output-path $output_dir/pano_gamma_compressed/ --output-format images --image-format exr + +conda activate everlight_new + +python lantern_scripts/un_gamma.py --data_dir $output_dir/pano_gamma_compressed/ --out_dir $output_dir/pano/ + +conda activate nerfstudio-HDR + +blender --background lantern_scripts/scene_new_my_apt.blend -P lantern_scripts/hdr_blender.py -- $output_dir/pano/ $output_dir/render/ \ No newline at end of file