diff --git a/README-pandora.md b/README-pandora.md index 4e75ccf23..6d205efa9 100644 --- a/README-pandora.md +++ b/README-pandora.md @@ -4,15 +4,23 @@ ``` tmux new -s nerf_studio + docker run -it --rm -v /gel/usr/{USER_NAME}:/mnt/workspace/ -v /home-local2/{USER_NAME}.extra.nobkp/:/mnt/data/ --gpus '"device=0"' -p 8008:8008 -p 6006:6006 lantern_docker:latest /bin/bash + cd /mnt/workspace/lantern/nerfstudio-HDR git clone https://github.com/MrKarimiD/nerfstudio-HDR.git conda activate nerfstudio-HDR export PYTHONPATH=$pwd:$PYTHONPATH pip install -e . -pip install skylibs equilib scikit-surgerycore pydub +pip install scikit-surgerycore pydub skylibs piq OpenEXR Imath equilib + pip install -U git+https://github.com/luca-medeiros/lang-segment-anything.git +If lang-sam not installed, do the following... +``` +cd ../lang-segment-anything +pip install . +cd ../nerfstudio-HDR ``` If there is an issue with equilib, do the following... @@ -46,7 +54,8 @@ cd ../nerfstudio-HDR - Shutter speed: determined below - ISO: 800 - WB: 3500 - - Make sure the cameras point up to avoid having lights in the seam. + - Make sure the cameras point up to avoid having lights in the seam. + - Make sure to film up and down to get good coverage 2. First capture with well-exposed cameras. - Change shutter speed to get a well-exposed image. **Note the shutter speed.** @@ -54,7 +63,19 @@ cd ../nerfstudio-HDR - Clap at the beginning and at the end to use for synchronization. - Go around the scene with the two cameras well-exposed. -3. Second capture with Camera 1 (left) well-exposed and Camera 2 (right) fast-exposed. Change shutter speed of Camera 2 to 1/25000 (fastest exposure) and repeat the steps above +3. Second capture with Camera 1 (left) well-exposed and Camera 2 (right) fast-exposed. Change shutter speed of Camera 2 to 1/25000 (fastest exposure) and repeat the steps above + + If calculating **METRICS**: Capture GT HDR brakets + - Go to settings and change the camera mode shooting method to multiple brackets + - Set the camera parameters: + - Apreture: 2.1 + - Shutter speed: Calculate the 11 shutter speeds and select the closest ones on the app in order from smallest to hightes + ``` + python lantern_scripts/calculate_exposures.py --min 0.00004 --max 0.004 + ``` + - ISO: 800 + - WB: 3500 + - Take 10 different locations 4. Import the files on computer. If on a mac, use Ricoh Theta File Transfer for Mac @@ -65,11 +86,21 @@ cd ../nerfstudio-HDR - Camera 1: - First capture: left_sfm - Second capture: left_e1 - - Camera 2: + - Camera 2 (fast exposed): - First capture: right_sfm - Second capture: right_e2 -6. Transfer videos on lab machine. + If calculating **METRICS**: Group GT brackets under GT folder like so + GT/ + GT1/ + 001.jpg + 002.jpg + ... + GT2/ + ... + ... + +6. Transfer files on lab machine. ``` scp -r /path/to/source/file /path/to/destination/ @@ -77,10 +108,20 @@ cd ../nerfstudio-HDR 7. Process the videos for OpenSFM. Input the right shutter speed noted above. + If not calculating metrics: ``` python lantern_scripts/process_videos_for_sfm.py --input_dir /mnt/data/scene/ --shutter_speed 0.004 ``` + If calculating **METRICS**: + ``` + python lantern_scripts/process_videos_for_sfm.py --input_dir /mnt/data/scene/ --shutter_speed 0.004 --gt + ``` + - Resize GT_exr and GT_jpg images to 3840x1920 with GNU Image Manipulation Program + - Go to Image > Scale Image + - Transfer those back on lab machine in th GT folder of the scene naming them GT_jpg_small and GT_exr_small + - Add GT_jpg images into sfm/images/ with their appropriate mask (checkout stick_masks folder) + ## 2. Process data using OpenSFM ``` @@ -167,34 +208,45 @@ ns-eval --load-config=outputs/scene_ns/lantern-nerfacto/2024-06-17_152246/config ## 4. Calculate metrics +``` +python lantern_scripts/calculate_metrics.py --input_dir /mnt/data/coffee_room2/ --checkpoint /mnt/workspace/lantern/nerfstudio-HDR/outputs/coffee_room2_ns/lantern-nerfacto/2025-01-22_194346/ --metric_name unaligned +``` + +and point #8 + +or + 1. Get ground truth transformations: ``` - ns-process-data lantern-GT-HDR --data /mnt/data/lab_downstairs_ns/ --output-dir /mnt/data/metrics/lab_downstairs --metadata /mnt/data/lab_downstairs_ns/reconstruction.json --checkpoint /mnt/workspace/lantern/nerfstudio-HDR/outputs/lab_downstairs_ns/lantern-nerfacto/2024-10-06_232947/ + ns-process-data lantern-GT-HDR --data /mnt/data/coffee_room2/coffee_room2_ns/ --output-dir /mnt/data/coffee_room2/metrics --metadata /mnt/data/coffee_room2/sfm/reconstruction.json --checkpoint /mnt/workspace/lantern/nerfstudio-HDR/outputs/coffee_room2_ns/lantern-nerfacto/2025-01-22_194346/ ``` 2. Render panos at ground truth positions for well and fast exposed: ``` - ns-render camera-path --load-config /mnt/workspace/lantern/nerfstudio-HDR/outputs/lab_downstairs_ns/lantern-nerfacto/2024-10-06_232947/config.yml --camera-path-filename /mnt/data/metrics/lab_downstairs/GT_transforms.json --output-path /mnt/data/metrics/renders/lab_downstairs_aligned --output-format images --rendered_output_names + ns-render camera-path --load-config /mnt/workspace/lantern/nerfstudio-HDR/outputs/coffee_room2_ns/lantern-nerfacto/2025-01-22_194346/config.yml --camera-path-filename /mnt/data/coffee_room2/metrics/GT_transforms.json --output-path /mnt/data/coffee_room2/metrics/renders/unaligned --output-format images ``` ``` - ns-render camera-path --load-config /mnt/workspace/lantern/nerfstudio-HDR/outputs/lab_downstairs_ns/lantern-nerfacto/2024-10-06_232947/config.yml --camera-path-filename /mnt/data/metrics/lab_downstairs/GT_transforms.json --output-path /mnt/data/metrics/renders/lab_downstairs_aligned_fast --output-format images --rendered_output_names rgb_fast + ns-render camera-path --load-config /mnt/workspace/lantern/nerfstudio-HDR/outputs/coffee_room2_ns/lantern-nerfacto/2025-01-22_194346/config.yml --camera-path-filename /mnt/data/coffee_room2/metrics/GT_transforms.json --output-path /mnt/data/coffee_room2/metrics/renders/unaligned_fast --output-format images --rendered_output_names rgb_fast ``` 3. Combine both well and fast exposed panos: ``` - python lantern_scripts/combine.py --well_dir /mnt/data/metrics/renders/lab_downstairs_aligned/ --fast_dir /mnt/data/metrics/renders/lab_downstairs_aligned_fast/ --out_dir /mnt/data/metrics/renders/lab_downstairs_lin/ --experiment_location /mnt/data/lab_downstairs_ns/ --do_linearization + python lantern_scripts/combine.py --well_dir /mnt/data/coffee_room2/metrics/renders/unaligned/ --fast_dir /mnt/data/coffee_room2/metrics/renders/unaligned_fast/ --out_dir /mnt/data/coffee_room2/metrics/renders/unaligned_lin/ --experiment_location /mnt/data/coffee_room2/data/ --do_linearization ``` 4. Make sure the panos from renders have the same names as the GT panos. 5. Calculate PSNR, SSIM and LPIPS: ``` - python lantern_scripts/ldr_res.py --gt_dir /mnt/data/metrics/GT_lab_downstairs/pano/ --data_dir /mnt/data/metrics/renders/lab_downstairs_lin/ + python lantern_scripts/ldr_res.py --gt_dir /mnt/data/coffee_room2/GT/GT_exr/ --data_dir /mnt/data/coffee_room2/metrics/renders/unaligned_lin/ ``` -5. Generate HDR renders with blender: +5. Generate GT and results renders with blender: ``` - blender --background /mnt/data/metrics/scene_new.blend -P lantern_scripts/hdr_blender.py -- /mnt/data/metrics/renders/lab_downstairs_lin_less/ /mnt/data/metrics/renders/lab_downstairs_lin_renders/ + blender --background lantern_scripts/scene_new.blend -P lantern_scripts/hdr_blender.py -- /mnt/data/coffee_room2/GT/GT_exr/ /mnt/data/coffee_room2/GT/GT_exr_renders/ + ``` + ``` + blender --background lantern_scripts/scene_new.blend -P lantern_scripts/hdr_blender.py -- /mnt/data/coffee_room2/metrics/renders/unaligned_lin/ /mnt/data/coffee_room2/metrics/renders/unaligned_lin_renders/ ``` 6. Calculate si-RMSE, RMSE, RGB ang. and PSNR (LDR r.). Need to change values in code: diff --git a/lantern_scripts/align_fast_to_well.py b/lantern_scripts/align_fast_to_well.py index 5cfd48d00..1966ca958 100644 --- a/lantern_scripts/align_fast_to_well.py +++ b/lantern_scripts/align_fast_to_well.py @@ -125,6 +125,20 @@ class AlignImages: iterations += 1 return list(zip(self.matched_well_contours, self.fast_contours)) + def match_contours_simplified(self): #something to try out + for contour_index, fast_contour in enumerate(self.fast_contours): + for threshold_index, _ in enumerate(self.well_thresholds): + for well_contour in self.well_images[threshold_index].contours: + try: + well_contour = Contour(well_contour, threshold_index) + score = fast_contour.get_matching_score(well_contour) + if score < fast_contour.score and score <= 200: + fast_contour.match_contour(well_contour, score) + self.matched_well_contours[contour_index] = well_contour + except ContourTooSmallException as e: + continue + return list(zip(self.matched_well_contours, self.fast_contours)) + def get_point_descriptors(self, points, image): sift = cv2.SIFT_create() keypoints = [cv2.KeyPoint(x=float(p[0]), y=float(p[1]), size=20) for p in points] @@ -233,8 +247,8 @@ if __name__ == '__main__': argparser = argparse.ArgumentParser() argparser.add_argument("--input_dir", type=str, default="/mnt/data/scene/scene_ns/", help="Nerfstudio directory of the scene") argparser.add_argument("--config", type=str, default="outputs/scene_ns/lantern-nerfacto/2024-06-20_142413/config.yml", help="Nerfstudio config file") - argparser.add_argument("--no_render", action='store_true', default="False", help="Doesn't render images for alignment if flag is present") - argparser.add_argument("--no_copy", action='store_true', default="Flase", help="Doesn't copy the original data if flag is present. Use if already copied.") + argparser.add_argument("--no_render", action='store_true', default=False, help="Doesn't render images for alignment if flag is present") + argparser.add_argument("--no_copy", action='store_true', default=False, help="Doesn't copy the original data if flag is present. Use if already copied.") args = argparser.parse_args() json_path = os.path.join(args.input_dir, 'alignment_matrices.json') diff --git a/lantern_scripts/calculate_exposures.py b/lantern_scripts/calculate_exposures.py new file mode 100644 index 000000000..d024a89e7 --- /dev/null +++ b/lantern_scripts/calculate_exposures.py @@ -0,0 +1,21 @@ +import numpy as np +import argparse + +def calculate_exposures(min_exposure, max_exposure, nb_exposures): + return 2 ** np.linspace(np.log2(min_exposure), np.log2(max_exposure), nb_exposures) + +if __name__ == "__main__": + argparser = argparse.ArgumentParser() + argparser.add_argument("--min", type=float, required=True, default=0.00004) + argparser.add_argument("--max", type=float, required=True, default=0.004) + args = argparser.parse_args() + + min_exposure = args.min + max_exposure = args.max + nb_exposures = 11 # min and max exposures included + + exposures = calculate_exposures(min_exposure, max_exposure, nb_exposures) + print("Calculated exposures:") + for exposure in reversed(exposures): + denominator = round(1 / exposure) + print(f"1/{denominator}") \ No newline at end of file diff --git a/lantern_scripts/calculate_metrics.py b/lantern_scripts/calculate_metrics.py new file mode 100644 index 000000000..8f65b89e1 --- /dev/null +++ b/lantern_scripts/calculate_metrics.py @@ -0,0 +1,160 @@ +import subprocess +import argparse +import os +import shutil + +if __name__ == '__main__': + argparser = argparse.ArgumentParser() + argparser.add_argument("--input_dir", type=str, required=True, default="/mnt/data/scene/scene_ns/", + help="Directory containing input data (e.g. videos, GT, etc.)") + argparser.add_argument("--checkpoint", type=str, required=True, + default="/mnt/workspace/lantern/nerfstudio-HDR/outputs/coffee_room2_ns/lantern-nerfacto/2025-01-22_194346/", + help="Directory containing training data (e.g. config.yml and checkpoint info)") + argparser.add_argument("--metric_name", type=str, required=True, default="unaligned", + help="Name of the method, used for naming output directories") + + args = argparser.parse_args() + + if not args.input_dir.endswith('/'): + args.input_dir += '/' + + metrics_dir = os.path.join(args.input_dir, "metrics") + renders_dir = os.path.join(metrics_dir, "images") + renders_well = os.path.join(renders_dir, args.metric_name) + renders_fast = os.path.join(renders_dir, args.metric_name + "_fast") + renders_lin = os.path.join(renders_dir, args.metric_name + "_lin") + renders_lin_HDR = os.path.join(renders_dir, args.metric_name + "_lin_HDR") + lin_renders = os.path.join(renders_dir, args.metric_name + "_lin_renders") + + gt_dir = os.path.join(args.input_dir, "GT") + gt_hdr = os.path.join(gt_dir, "GT_hdr") + gt_exr = os.path.join(gt_dir, "GT_exr") + gt_exr_renders = os.path.join(gt_dir, "GT_exr_renders") + + # 1. Get ground truth transformations: + print("Running ns-process-data to get GT transformations...") + subprocess.run([ + "ns-process-data", "lantern-GT-HDR", + "--data", args.input_dir, + "--output-dir", metrics_dir, + "--metadata", os.path.join(args.input_dir, "sfm/reconstruction.json"), + "--checkpoint", args.checkpoint + ], check=True) + + # 2. Render panos at GT positions for well and fast exposures: + print("Rendering well-exposed panos...") + subprocess.run([ + "ns-render", "camera-path", + "--load-config", os.path.join(args.checkpoint, "config.yml"), + "--camera-path-filename", os.path.join(metrics_dir, "GT_transforms.json"), + "--output-path", renders_well, + "--output-format", "images" + ], check=True) + + print("Rendering fast-exposed panos...") + subprocess.run([ + "ns-render", "camera-path", + "--load-config", os.path.join(args.checkpoint, "config.yml"), + "--camera-path-filename", os.path.join(metrics_dir, "GT_transforms.json"), + "--output-path", renders_fast, + "--output-format", "images", + "--rendered_output_names", "rgb_fast" + ], check=True) + + # 3. Combine both well and fast exposed panos: + print("Combining well and fast exposure renders...") + subprocess.run([ + "python", "lantern_scripts/combine.py", + "--well_dir", renders_well + "/", + "--fast_dir", renders_fast + "/", + "--out_dir", renders_lin + "/", + "--experiment_location", os.path.join(args.input_dir, "data") + "/", + "--do_linearization" + ], check=True) + + # 4. Rename the render files: + print("Step 4a: Renaming render filenames for consistency between GT and renders...") + render_files = sorted([f for f in os.listdir(renders_lin) if f.endswith('.exr')]) + print("Render files found:", render_files) + total = len(render_files) + number_strings = [str(i) for i in range(1, total + 1)] + lex_order = sorted(number_strings) + for old_path, num in zip(render_files, lex_order): + new_filename = f"GT{num}.exr" + new_path = os.path.join(renders_lin, new_filename) + old_path = os.path.join(renders_lin, old_path) + shutil.move(old_path, new_path) + print(f"Renamed {os.path.basename(old_path)} to {new_filename}") + + + print("Step 4b: Verifying filename consistency between GT and renders...") + try: + gt_files = set(f for f in os.listdir(gt_exr) if f.endswith('.exr')) + render_files = set(f for f in os.listdir(renders_lin) if f.endswith('.exr')) + missing_in_renders = gt_files - render_files + extra_in_renders = render_files - gt_files + if missing_in_renders or extra_in_renders: + if missing_in_renders: + print("Error: The following GT files are missing in the renders directory:", missing_in_renders) + if extra_in_renders: + print("Error: There are extra files in the renders directory not found in GT:", extra_in_renders) + sys.exit(1) + else: + print("All GT filenames are present in the renders directory.") + except Exception as e: + print("Error during filename verification:", e) + sys.exit(1) + + # 5. Calculate PSNR, SSIM and LPIPS: + print("Calculating LDR metrics (PSNR, SSIM, LPIPS)...") + subprocess.run([ + "python", "lantern_scripts/ldr_res.py", + "--gt_dir", gt_exr + "/", + "--data_dir", renders_lin + "/" + ], check=True) + + # 6. Generate GT and result renders with Blender: + expected_render_dir = gt_exr_renders + rendered_files_exist = any( + f.endswith('.exr') for f in os.listdir(expected_render_dir) + ) if os.path.exists(expected_render_dir) else False + + if not rendered_files_exist: + print("Rendering GT images with Blender...") + subprocess.run([ + "blender", "--background", "lantern_scripts/scene_new.blend", + "-P", "lantern_scripts/hdr_blender.py", "--", + os.path.join(gt_exr, ""), os.path.join(gt_exr_renders, "") + ], check=True) + else: + print("Skipping Blender render — output files already exist.") + + print("Rendering result images with Blender...") + subprocess.run([ + "blender", "--background", "lantern_scripts/scene_new.blend", + "-P", "lantern_scripts/hdr_blender.py", "--", + renders_lin + "/", lin_renders + "/" + ], check=True) + + # 7. Calculate si-RMSE, RMSE, RGB angular error and PSNR (LDR rendered): + print("Calculating additional metrics (si-RMSE, RMSE, angular error, PSNR)...") + subprocess.run([ + "python", "lantern_scripts/res_table_1.py", + "--results_dataset_roots", lin_renders + "/", + "--gt_dataset_root", gt_exr_renders + "/", + "--output_folder", metrics_dir + "/results_table/", + "--dataset_names", args.metric_name + ], check=True) + + # 8. Convert EXR panos to HDR format: + print("Converting EXR panos to HDR...") + subprocess.run([ + "python", "lantern_scripts/exr2hdr.py", + "--hdr_dir", renders_lin, + "--output_dir", renders_lin_HDR + ], check=True) + subprocess.run([ + "python", "lantern_scripts/exr2hdr.py", + "--hdr_dir", gt_exr, + "--output_dir", gt_hdr + ], check=True) diff --git a/lantern_scripts/create_GT_folder.py b/lantern_scripts/create_GT_folder.py new file mode 100644 index 000000000..d961c1397 --- /dev/null +++ b/lantern_scripts/create_GT_folder.py @@ -0,0 +1,45 @@ +import os +import shutil +import re +import argparse + +def copy_files(input_dir): + brackets = os.listdir(input_dir) + + target_exr = os.path.join(input_dir, "GT_exr") + target_jpg = os.path.join(input_dir, "GT_jpg") + os.makedirs(target_exr, exist_ok=True) + os.makedirs(target_jpg, exist_ok=True) + + for folder in brackets: + if not folder.startswith("GT_"): + folder_path = os.path.join(input_dir, folder) + if os.path.isdir(folder_path): + smallest_value = None + smallest_file = None + for file in os.listdir(folder_path): + if file.startswith("GT") and file.lower().endswith(".exr"): + source_file = os.path.join(folder_path, file) + destination_file = os.path.join(target_exr, file) + shutil.copy2(source_file, destination_file) + print(f"Copied EXR: {source_file} -> {destination_file}") + + if file.lower().endswith(".jpg"): + numbers = re.findall(r'\d+', file) + num = int(numbers[0]) + if smallest_value is None or num < smallest_value: + smallest_value = num + smallest_file = file + + source_file = os.path.join(folder_path, smallest_file) + destination_file = os.path.join(target_jpg, f"{folder}.jpg") + shutil.copy2(source_file, destination_file) + print(f"Copied JPG: {source_file} -> {destination_file}") + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--input_dir', type=str, required=True, + help='GT directory containing all the subdirectories (e.g., GT1, GT2, ...) with frames') + args = parser.parse_args() + + copy_files(args.input_dir) diff --git a/lantern_scripts/create_data_folder.py b/lantern_scripts/create_data_folder.py index 6372df618..f1a147b67 100644 --- a/lantern_scripts/create_data_folder.py +++ b/lantern_scripts/create_data_folder.py @@ -3,22 +3,6 @@ import shutil import json import argparse -def move_subfolder(source_dir, destination_dir): - if not os.path.exists(source_dir): - print(f"Source directory '{source_dir}' does not exist.") - return - - os.makedirs(destination_dir, exist_ok=True) - - try: - for filename in os.listdir(source_dir): - if filename.lower().count('png') < 2: - shutil.move(os.path.join(source_dir, filename), os.path.join(destination_dir, filename)) - except Exception as e: - print(f"Error moving subfolder: {e}") - - delete_empty_folder(source_dir) - def copy_subfolder(source_dir, destination_dir): if not os.path.exists(source_dir): print(f"Source directory '{source_dir}' does not exist.") @@ -33,17 +17,6 @@ def copy_subfolder(source_dir, destination_dir): except Exception as e: print(f"Error copying subfolder: {e}") -def delete_empty_folder(folder_path): - try: - if os.path.exists(folder_path): - if not os.listdir(folder_path): - os.rmdir(folder_path) - print(f"Folder '{folder_path}' deleted successfully.") - else: - print(f"Folder '{folder_path}' does not exist.") - except Exception as e: - print(f"Error deleting folder: {e}") - def create_capture_settings_file(file_path, shutter_speed): capture_settings = { "left": { @@ -66,7 +39,7 @@ def create_capture_settings_file(file_path, shutter_speed): if __name__ == "__main__": argparser = argparse.ArgumentParser() - argparser.add_argument("--input_dir", type=str, default="/mnt/data/scene/") + argparser.add_argument("--input_dir", type=str, default="/mnt/data/scene/data/") argparser.add_argument("--shutter_speed", type=float, default=0.003125) args = argparser.parse_args() @@ -76,8 +49,6 @@ if __name__ == "__main__": for item in os.listdir(masks_folder): copy_subfolder(masks_folder + item, dest_folder + item + "/mask") - delete_empty_folder(args.input_dir + "masks") - create_capture_settings_file(dest_folder + "capture_settings.json", args.shutter_speed) diff --git a/lantern_scripts/exr2hdr.py b/lantern_scripts/exr2hdr.py new file mode 100644 index 000000000..9e42048f5 --- /dev/null +++ b/lantern_scripts/exr2hdr.py @@ -0,0 +1,44 @@ +import os +import cv2 +import argparse +import numpy as np +from tqdm import tqdm +import numpy as np +from pathlib import Path +from hdrio import imwrite +import imageio + + +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" +imageio.plugins.freeimage.download() + +def convert(src_addr, trg_addr): + img = imageio.imread(src_addr) + imwrite( img[:, :, 0:3], trg_addr) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--hdr_dir', type=str) + parser.add_argument('--output_dir', type=str) + parser.add_argument('--change_name', action='store_true') + args = parser.parse_args() + + hdrs = [] + for path in Path(args.hdr_dir).glob('*.exr'): + if path.is_file(): + hdrs.append(str(path)) + if len(hdrs) == 0: + raise Exception("No data available!!!") + hdrs = sorted(hdrs) + + os.makedirs(args.output_dir, exist_ok=True) + + for index, src_addr in enumerate(tqdm(hdrs)): + src_addr = str(src_addr) + if args.change_name: + trg_addr = str(Path(args.output_dir) / f"{index:05d}.hdr") + else: + trg_addr = src_addr.replace(args.hdr_dir, args.output_dir) + trg_addr = trg_addr.replace('.exr', '.hdr') + convert(src_addr, trg_addr) diff --git a/lantern_scripts/ldr_res.py b/lantern_scripts/ldr_res.py index 7e614f3a2..84e64d999 100755 --- a/lantern_scripts/ldr_res.py +++ b/lantern_scripts/ldr_res.py @@ -80,4 +80,17 @@ if __name__ == "__main__": print("SSIM: ", ssim_index) print("PSNR: ", psnr_index) lpips_index = piq.LPIPS(reduction='none')(torch.stack(gts), torch.stack(preds)) - print("LPIPS: ", torch.mean(lpips_index)) \ No newline at end of file + print("LPIPS: ", torch.mean(lpips_index)) + + results = { + "SSIM": str(ssim_index), + "PSNR": str(psnr_index), + "LPIPS": str(torch.mean(lpips_index)) + } + + output_path = os.path.join(args.data_dir, "ldr_results.json") + + with open(output_path, 'w') as f: + json.dump(results, f, indent=4) + + print(f"Saved metrics to: {output_path}") \ No newline at end of file diff --git a/lantern_scripts/process_videos_for_sfm.py b/lantern_scripts/process_videos_for_sfm.py index f4bda5575..8a6941249 100644 --- a/lantern_scripts/process_videos_for_sfm.py +++ b/lantern_scripts/process_videos_for_sfm.py @@ -5,10 +5,16 @@ if __name__ == '__main__': argparser = argparse.ArgumentParser() argparser.add_argument("--input_dir", type=str, required=True, default="/mnt/data/scene/", help="Directory containing all the 4 videos") argparser.add_argument("--shutter_speed", type=float, required=True, default=0.003125) + argparser.add_argument("--gt", action='store_true', default=False, help="Process GT images for metrics if flag is present") args = argparser.parse_args() - subprocess.run(["python", "lantern_scripts/sync_and_trim.py", "--input_dir", args.input_dir]) - subprocess.run(["python", "lantern_scripts/mp4_videos_to_png_frames.py", "--input_dir", args.input_dir + "data/"]) - subprocess.run(["python", "lantern_scripts/mask_humans2.py", "--input_dir", args.input_dir + "data/", "--output_dir", args.input_dir + "data/masks"]) - subprocess.run(["python", "lantern_scripts/create_data_folder.py", "--input_dir", args.input_dir + "data/", "--shutter_speed", str(args.shutter_speed)]) - subprocess.run(["python", "lantern_scripts/create_sfm_folder.py", "--input_dir", args.input_dir]) + subprocess.run(["python", "lantern_scripts/sync_and_trim.py", "--input_dir", args.input_dir], check=True) + subprocess.run(["python", "lantern_scripts/mp4_videos_to_png_frames.py", "--input_dir", args.input_dir + "data/"], check=True) + subprocess.run(["python", "lantern_scripts/mask_humans2.py", "--input_dir", args.input_dir + "data/", "--output_dir", args.input_dir + "data/masks"], check=True) + subprocess.run(["python", "lantern_scripts/create_data_folder.py", "--input_dir", args.input_dir + "data/", "--shutter_speed", str(args.shutter_speed)], check=True) + subprocess.run(["python", "lantern_scripts/create_sfm_folder.py", "--input_dir", args.input_dir], check=True) + + # For GT + if args.gt: + subprocess.run(["python", "lantern_scripts/debevec_bracket_merge.py", "--data_dir", args.input_dir + "GT/", "--experiment_location", args.input_dir + "data/"], check=True) + subprocess.run(["python", "lantern_scripts/create_GT_folder.py", "--input_dir", args.input_dir + "GT/"], check=True) diff --git a/lantern_scripts/render_mask.png b/lantern_scripts/render_mask.png new file mode 100755 index 000000000..22b70fe1b Binary files /dev/null and b/lantern_scripts/render_mask.png differ diff --git a/lantern_scripts/res_table_1.py b/lantern_scripts/res_table_1.py index 7d391a2aa..02ce80244 100755 --- a/lantern_scripts/res_table_1.py +++ b/lantern_scripts/res_table_1.py @@ -6,6 +6,7 @@ from math import log10 import json import cv2 import imageio +import argparse def wrmse(gt, est, mask): @@ -84,104 +85,79 @@ def remove_nan_and_sort(x): x = np.sort(x) return x +def calculate_metrics(results_dataset_roots, gt_dataset_root, output_folder, dataset_names): + results_dataset_roots = [results_dataset_roots] + dataset_names = [dataset_names] + mse = {} + si_mse = {} + angular = {} + psnr_error = {} + for key in dataset_names: + mse[key] = [] + si_mse[key] = [] + angular[key] = [] + psnr_error[key] = [] -results_dataset_roots = [ "/mnt/data/metrics/renders/lab_downstairs_lin_renders/" - # '/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/LDR-Nerfacto/render/', - # '/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/linearization_then_nerf/render/', - # '/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/nerf_then_linearization/render/', - # '/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/HDR-Nerfacto/render/', - # '/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/PanoHDR-NeRF/render/' - ] -# results_dataset_roots = '/home/karimida/evaluation/final_results/Henrique_data/my_renderings_of_Henrique_2' + gt_dataset_files = sorted([os.path.join(gt_dataset_root, f) for f in os.listdir(gt_dataset_root) if f.endswith('.exr')]) -dataset_names = [ "lab_downstairs_lin" - # 'LDR_nerfacto', - # 'linearization_then_NeRF', - # 'NeRF_then_linearization', - # 'HDR-Nerfacto', - # 'PanoHDRNeRF' - ] + # create output folder output_folder, 'metrics' + os.system('mkdir -p ' + output_folder) -mse = {} -si_mse = {} -angular = {} -psnr_error = {} -for key in dataset_names: - mse[key] = [] - si_mse[key] = [] - angular[key] = [] - psnr_error[key] = [] - -# Ground truth for the indoor -# gt_dataset_root = '/home/karimida/evaluation/redoing_table_one/hdr_env_maps/gt_render_diffuse/' - -# Ground truth for the outdoor -# gt_dataset_root = '/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/GT/render/' -gt_dataset_root = '/mnt/data/metrics/GT_lab_downstairs/render/' -# output_folder = '/Users/momo/Desktop' -output_folder = '/mnt/data/metrics/lab_downstairs/res_table/' - -gt_dataset_files = sorted([os.path.join(gt_dataset_root, f) for f in os.listdir(gt_dataset_root) if f.endswith('.exr')]) - -mask_dataset_root = '/mnt/data/metrics/GT_lab_downstairs/render_mask/' - -# create output folder output_folder, 'metrics' -os.system('mkdir -p ' + output_folder + '/metrics') - -for results_dataset_root, dataset_name in zip(results_dataset_roots, dataset_names): - - for gt_dataset_file in tqdm(gt_dataset_files): - gt_dataset_img_exr = imread(gt_dataset_file)[:, :, :3].astype(np.float32) - - result_dataset_file = gt_dataset_file.replace(gt_dataset_root, results_dataset_root) - result_dataset_img_exr = imread(result_dataset_file)[:, :, :3].astype(np.float32) - - mask_file = gt_dataset_file.replace(gt_dataset_root, mask_dataset_root) - mask_file = mask_file.replace('.exr', '.png') - mask = np.float32(imageio.imread(mask_file)) / 255.0 - mask = mask == 1 - mask = np.repeat(mask[:, :, np.newaxis], 3, axis=2) - gt_dataset_img_exr_for_PSNR = gt_dataset_img_exr[mask] - result_dataset_img_exr_for_PSNR = result_dataset_img_exr[mask] + for results_dataset_root, dataset_name in zip(results_dataset_roots, dataset_names): - mse_result = wrmse(gt_dataset_img_exr, result_dataset_img_exr, mask) - scale_invariant_mse_result = si_wrmse(gt_dataset_img_exr, result_dataset_img_exr, mask) - angular_result = angular_error(gt_dataset_img_exr, result_dataset_img_exr, mask) - psnr_result = psnr(gt_dataset_img_exr_for_PSNR, result_dataset_img_exr_for_PSNR) - mse[dataset_name].append(mse_result) - si_mse[dataset_name].append(scale_invariant_mse_result) - angular[dataset_name].append(angular_result) - psnr_error[dataset_name].append(psnr_result) - si_mse[dataset_name].append(scale_invariant_mse_result) + for gt_dataset_file in tqdm(gt_dataset_files): + gt_dataset_img_exr = imread(gt_dataset_file)[:, :, :3].astype(np.float32) + + result_dataset_file = gt_dataset_file.replace(gt_dataset_root, results_dataset_root) + result_dataset_img_exr = imread(result_dataset_file)[:, :, :3].astype(np.float32) + mask = np.float32(imageio.imread('lantern_scripts/render_mask.png')) / 255.0 + mask = mask == 1 + mask = np.repeat(mask[:, :, np.newaxis], 3, axis=2) + gt_dataset_img_exr_for_PSNR = gt_dataset_img_exr[mask] + result_dataset_img_exr_for_PSNR = result_dataset_img_exr[mask] + + mse_result = wrmse(gt_dataset_img_exr, result_dataset_img_exr, mask) + scale_invariant_mse_result = si_wrmse(gt_dataset_img_exr, result_dataset_img_exr, mask) + angular_result = angular_error(gt_dataset_img_exr, result_dataset_img_exr, mask) + psnr_result = psnr(gt_dataset_img_exr_for_PSNR, result_dataset_img_exr_for_PSNR) + mse[dataset_name].append(mse_result) + si_mse[dataset_name].append(scale_invariant_mse_result) + angular[dataset_name].append(angular_result) + psnr_error[dataset_name].append(psnr_result) + si_mse[dataset_name].append(scale_invariant_mse_result) + mse[dataset_name] = remove_nan_and_sort(mse[dataset_name]) - # mse = {key: remove_nan_and_sort(mse[key]) for key in mse} - mse[dataset_name] = remove_nan_and_sort(mse[dataset_name]) + mse_ours = np.mean(mse[dataset_name]) + si_mse_ours = np.mean(si_mse[dataset_name]) + angular_ours = np.mean(angular[dataset_name]) + psnr_ours = np.mean(psnr_error[dataset_name]) - # mse_ours = np.percentile(mse[dataset_name], q=50) - # si_mse_ours = np.percentile(si_mse[dataset_name], q=50) - # angular_ours = np.percentile(angular[dataset_name], q=50) - # psnr_ours = np.percentile(psnr_error[dataset_name], q=50) + # print(dataset_name + ', si_RMSE: {}'.format(si_mse_ours)) + # print(dataset_name + ', RMSE: {}'.format(mse_ours)) + # print(dataset_name + ', angular: {}'.format(angular_ours)) + # print(dataset_name + ', PSNR: {}'.format(psnr_ours)) - mse_ours = np.mean(mse[dataset_name]) - si_mse_ours = np.mean(si_mse[dataset_name]) - angular_ours = np.mean(angular[dataset_name]) - psnr_ours = np.mean(psnr_error[dataset_name]) + output_dict = {} + output_dict['model_name'] = dataset_name + output_dict['si_RMSE'] = str(si_mse_ours) + output_dict['RMSE'] = str(mse_ours) + output_dict['angular'] = str(angular_ours) + output_dict['PSNR'] = str(psnr_ours) - # print(dataset_name + ', si_RMSE: {}'.format(si_mse_ours)) - # print(dataset_name + ', RMSE: {}'.format(mse_ours)) - # print(dataset_name + ', angular: {}'.format(angular_ours)) - # print(dataset_name + ', PSNR: {}'.format(psnr_ours)) + output_addr = output_folder + 'render_results.json' - output_dict = {} - output_dict['model_name'] = dataset_name - output_dict['si_RMSE'] = str(si_mse_ours) - output_dict['RMSE'] = str(mse_ours) - output_dict['angular'] = str(angular_ours) - output_dict['PSNR'] = str(psnr_ours) + with open(output_addr, 'w') as f: + json.dump(output_dict, f, indent=4) - output_addr = output_folder + '/metrics/' + dataset_name + '_results.json' - # print("output_addr: ", output_addr) - with open(output_addr, 'w') as f: - json.dump(output_dict, f, indent=4) \ No newline at end of file +if __name__ == '__main__': + argparser = argparse.ArgumentParser() + argparser.add_argument("--results_dataset_roots", type=str, default="/mnt/data/coffee_room2/metrics/renders/unaligned_lin_renders/") + argparser.add_argument("--gt_dataset_root", type=str, default="/mnt/data/coffee_room2/GT/GT_exr_renders/") + argparser.add_argument("--output_folder", type=str, default="/mnt/data/coffee_room2/metrics/results_table/") + argparser.add_argument("--dataset_names", type=str, default="/mnt/data/coffee_room2/metrics/results_table/") + + args = argparser.parse_args() + + calculate_metrics(args.results_dataset_roots, args.gt_dataset_root, args.output_folder, args.dataset_names) \ No newline at end of file diff --git a/nerfstudio/process_data/GT_HDR_to_nerfstudio.py b/nerfstudio/process_data/GT_HDR_to_nerfstudio.py index 47e798d17..6c6ce5009 100644 --- a/nerfstudio/process_data/GT_HDR_to_nerfstudio.py +++ b/nerfstudio/process_data/GT_HDR_to_nerfstudio.py @@ -113,6 +113,7 @@ class GTHDRoNeRFStudioDataset(BaseConverterToNerfstudioDataset): camera_dict["fps"] = 1 camera_key = 'v2 unknown unknown -1 -1 perspective 0' # camera_key = 'v2 unknown unknown 3840 1920 perspective 0.0' + # camera_key = 'v2 unknown unknown 6720 3360 perspective 0.0' camera_dict["camera_type"] = "equirectangular" camera_dict["render_width"] = openSFM_reconstruction[0]["cameras"][camera_key]['width'] camera_dict["render_height"] = openSFM_reconstruction[0]["cameras"][camera_key]['height'] diff --git a/stick_masks/GT_center_mask.png b/stick_masks/GT_center_mask.png new file mode 100755 index 000000000..f5c0ab2a4 Binary files /dev/null and b/stick_masks/GT_center_mask.png differ diff --git a/stick_masks/GT_side_mask.png b/stick_masks/GT_side_mask.png new file mode 100755 index 000000000..c54760dd7 Binary files /dev/null and b/stick_masks/GT_side_mask.png differ