diff --git a/lantern_scripts/calculate_metrics_panohdr_nerfacto.py b/lantern_scripts/calculate_metrics_panohdr_nerfacto.py new file mode 100644 index 000000000..99a3702f9 --- /dev/null +++ b/lantern_scripts/calculate_metrics_panohdr_nerfacto.py @@ -0,0 +1,198 @@ +import subprocess +import argparse +import os +import shutil +import sys + +if __name__ == '__main__': + argparser = argparse.ArgumentParser() + argparser.add_argument("--input_dir", type=str, required=True, default="/mnt/data/scene/scene_ns/", + help="Directory containing input data (e.g. videos, GT, etc.)") + argparser.add_argument("--checkpoint", type=str, required=True, + default="/mnt/workspace/lantern/nerfstudio-HDR/outputs/coffee_room2_ns/lantern-nerfacto/2025-01-22_194346/", + help="Directory containing training data (e.g. config.yml and checkpoint info)") + argparser.add_argument("--metric_name", type=str, required=True, default="unaligned", + help="Name of the method, used for naming output directories") + + args = argparser.parse_args() + + if not args.input_dir.endswith('/'): + args.input_dir += '/' + + metrics_dir = os.path.join(args.input_dir, "metrics") + renders_dir = os.path.join(metrics_dir, "images") + + renders_compressed = os.path.join(renders_dir, args.metric_name + "_compressed") + if os.path.exists(renders_compressed): + shutil.rmtree(renders_compressed) + + renders_lin = os.path.join(renders_dir, args.metric_name + "_lin") + if os.path.exists(renders_lin): + shutil.rmtree(renders_lin) + + renders_lin_HDR = os.path.join(renders_dir, args.metric_name + "_lin_HDR") + if os.path.exists(renders_lin_HDR): + shutil.rmtree(renders_lin_HDR) + + lin_renders = os.path.join(renders_dir, args.metric_name + "_lin_renders") + if os.path.exists(lin_renders): + shutil.rmtree(lin_renders) + + gt_dir = os.path.join(args.input_dir, "GT") + gt_hdr = os.path.join(gt_dir, "GT_hdr") + if os.path.exists(gt_hdr): + shutil.rmtree(gt_hdr) + gt_exr = os.path.join(gt_dir, "GT_exr") + gt_exr_renders = os.path.join(gt_dir, "GT_exr_renders") + if os.path.exists(gt_exr_renders): + shutil.rmtree(gt_exr_renders) + + # 1. Get ground truth transformations: + print("Running ns-process-data to get GT transformations...") + subprocess.run([ + "ns-process-data", "lantern-GT-HDR", + "--data", args.input_dir, + "--output-dir", metrics_dir, + "--metadata", os.path.join(args.input_dir, "sfm/reconstruction.json"), + "--checkpoint", args.checkpoint + ], check=True) + + # 2. Render panos at GT positions for well and fast exposures: + print("Rendering HDR panos...") + subprocess.run([ + "ns-render", "camera-path", + "--load-config", os.path.join(args.checkpoint, "config.yml"), + "--camera-path-filename", os.path.join(metrics_dir, "GT_transforms.json"), + "--output-path", renders_compressed, + "--output-format", "images", + "--image-format", "exr" + ], check=True) + + # 3. Combine both well and fast exposed panos: + print("Ungamma renders...") + subprocess.run([ + "python", "lantern_scripts/un_gamma.py", + "--data_dir", renders_compressed + "/", + "--out_dir", renders_lin + "/" + ], check=True) + + # 4. Rename the render files: + print("Step 4a: Renaming render filenames for consistency between GT and renders...") + render_files = sorted([f for f in os.listdir(renders_lin) if f.endswith('.exr')]) + print("Render files found:", render_files) + total = len(render_files) + number_strings = [str(i) for i in range(1, total + 1)] + lex_order = sorted(number_strings) + for old_path, num in zip(render_files, lex_order): + new_filename = f"GT{num}.exr" + new_path = os.path.join(renders_lin, new_filename) + old_path = os.path.join(renders_lin, old_path) + shutil.move(old_path, new_path) + print(f"Renamed {os.path.basename(old_path)} to {new_filename}") + + print("Step 4b: Verifying filename consistency between GT and renders...") + try: + gt_files = set(f for f in os.listdir(gt_exr) if f.endswith('.exr')) + render_files = set(f for f in os.listdir(renders_lin) if f.endswith('.exr')) + missing_in_renders = gt_files - render_files + extra_in_renders = render_files - gt_files + if missing_in_renders or extra_in_renders: + if missing_in_renders: + print("Error: The following GT files are missing in the renders directory:", missing_in_renders) + if extra_in_renders: + print("Error: There are extra files in the renders directory not found in GT:", extra_in_renders) + sys.exit(1) + else: + print("All GT filenames are present in the renders directory.") + except Exception as e: + print("Error during filename verification:", e) + sys.exit(1) + + # 5. Calculate PSNR, SSIM and LPIPS: + print("Calculating LDR metrics (PSNR, SSIM, LPIPS)...") + subprocess.run([ + "python", "lantern_scripts/ldr_res.py", + "--gt_dir", gt_exr + "/", + "--data_dir", renders_lin + "/" + ], check=True) + + # 6. Generate GT and result renders with Blender: + expected_render_dir = gt_exr_renders + rendered_files_exist = any( + f.endswith('.exr') for f in os.listdir(expected_render_dir) + ) if os.path.exists(expected_render_dir) else False + + if not rendered_files_exist: + print("Rendering GT images with Blender...") + subprocess.run([ + "blender", "--background", "lantern_scripts/scene_new.blend", + "-P", "lantern_scripts/hdr_blender.py", "--", + os.path.join(gt_exr, ""), os.path.join(gt_exr_renders, "") + ], check=True) + else: + print("Skipping Blender render — output files already exist.") + + print("Rendering result images with Blender...") + subprocess.run([ + "blender", "--background", "lantern_scripts/scene_new.blend", + "-P", "lantern_scripts/hdr_blender.py", "--", + renders_lin + "/", lin_renders + "/" + ], check=True) + + # 7. Calculate si-RMSE, RMSE, RGB angular error and PSNR (LDR rendered): + print("Calculating additional metrics (si-RMSE, RMSE, angular error, PSNR)...") + subprocess.run([ + "python", "lantern_scripts/res_table_1.py", + "--results_dataset_roots", lin_renders + "/", + "--gt_dataset_root", gt_exr_renders + "/", + "--output_folder", metrics_dir + "/results_table/", + "--dataset_names", args.metric_name + ], check=True) + + # 8. Convert EXR panos to HDR format: + print("Converting EXR panos to HDR...") + subprocess.run([ + "python", "lantern_scripts/exr2hdr.py", + "--hdr_dir", renders_lin, + "--output_dir", renders_lin_HDR + ], check=True) + subprocess.run([ + "python", "lantern_scripts/exr2hdr.py", + "--hdr_dir", gt_exr, + "--output_dir", gt_hdr + ], check=True) + + # 9. Convert EXR panos to LDR format: + print("Converting EXR panos to LDR visualization...") + subprocess.run([ + "python", "lantern_scripts/tonemap.py", + "--data_dir", renders_lin, + "--out_dir", renders_lin + ], check=True) + subprocess.run([ + "python", "lantern_scripts/tonemap.py", + "--data_dir", gt_exr, + "--out_dir", gt_exr + ], check=True) + subprocess.run([ + "python", "lantern_scripts/tonemap.py", + "--data_dir", lin_renders, + "--out_dir", lin_renders + ], check=True) + subprocess.run([ + "python", "lantern_scripts/tonemap.py", + "--data_dir", gt_exr_renders, + "--out_dir", gt_exr_renders + ], check=True) + + # 9. Convert EXR panos to LDR format: + print("Preparing the HTML visualization...") + subprocess.run([ + "python", "lantern_scripts/results_visualization.py", + "--gt_dir", gt_exr, + "--gt_render_dir", gt_exr_renders, + "--pandora_dir", renders_lin, + "--pandora_render_dir", lin_renders, + "--out_dir", args.input_dir, + "--name", args.metric_name + ], check=True) diff --git a/lantern_scripts/combine_LDR_HDR.py b/lantern_scripts/combine_LDR_HDR.py index 577efe288..56f84df45 100644 --- a/lantern_scripts/combine_LDR_HDR.py +++ b/lantern_scripts/combine_LDR_HDR.py @@ -3,30 +3,34 @@ import numpy as np from tqdm import tqdm from envmap import EnvironmentMap, rotation_matrix from hdrio import imwrite +import argparse -# Outdoor Table -LDR_dir = '/mnt/data/nerfstudio_ds/real_data/lab_downstairs_data/left_e1_linearized/' -HDR_dir = '/mnt/data/nerfstudio_ds/real_data/lab_downstairs_data/left_e1_ldr2hdr_network/' -output_dir = '/mnt/data/nerfstudio_ds/real_data/lab_downstairs_data/left_e1_ldr2hdr_network_high_res/' -imgFiles = [os.path.join(LDR_dir, f) for f in os.listdir(LDR_dir) if os.path.isfile(os.path.join(LDR_dir, f)) and f.endswith('.png')] -imgFiles.sort() +if __name__ == '__main__': + argparser = argparse.ArgumentParser() + argparser.add_argument("--ldr_dir", type=str, required=True, default="/mnt/data/scene/", help="LDR folder, high resolution") + argparser.add_argument("--hdr_dir", type=str, required=True, default="/mnt/data/scene/", help="LDR2HDR results folder, low resolution") + argparser.add_argument("--output_dir", type=str, required=True, default="/mnt/data/scene/", help="Output") + args = argparser.parse_args() -for ldr_addr in tqdm(imgFiles): - ldr_pano = EnvironmentMap(ldr_addr, 'latlong') - - hdr_addr = ldr_addr.replace('.png', '.exr') - hdr_addr = hdr_addr.replace(LDR_dir, HDR_dir) - assert os.path.exists(hdr_addr), "HDR file does not exists!!!" - hdr_pano = EnvironmentMap(hdr_addr, 'latlong') - hdr_pano.resize(1920) + imgFiles = [os.path.join(args.ldr_dir, f) for f in os.listdir(args.ldr_dir) if os.path.isfile(os.path.join(args.ldr_dir, f)) and f.endswith('.png')] + imgFiles.sort() - out_pano = ldr_pano.data # ** 2.2 - list_hdr = hdr_pano.data > 1.0 - print("list_hdr.shape: ", list_hdr.shape, "list_hdr.shape: ", ldr_pano.data.shape) - out_pano[list_hdr] = hdr_pano.data[list_hdr] - - out_addr = hdr_addr.replace(HDR_dir, output_dir) - imwrite(out_pano, out_addr) + for ldr_addr in tqdm(imgFiles): + ldr_pano = EnvironmentMap(ldr_addr, 'latlong') + + hdr_addr = ldr_addr.replace('.png', '.exr') + hdr_addr = hdr_addr.replace(args.ldr_dir, args.hdr_dir) + assert os.path.exists(hdr_addr), "HDR file does not exists!!!" + hdr_pano = EnvironmentMap(hdr_addr, 'latlong') + hdr_pano.resize(1920) -print("Done!") \ No newline at end of file + out_pano = ldr_pano.data # ** 2.2 + list_hdr = hdr_pano.data > 1.0 + print("list_hdr.shape: ", list_hdr.shape, "list_hdr.shape: ", ldr_pano.data.shape) + out_pano[list_hdr] = hdr_pano.data[list_hdr] + + out_addr = hdr_addr.replace(args.hdr_dir, args.output_dir) + imwrite(out_pano, out_addr) + + print("Done!") \ No newline at end of file diff --git a/lantern_scripts/linearize_jpgs.py b/lantern_scripts/linearize_jpgs.py index 797395a48..362e70a62 100755 --- a/lantern_scripts/linearize_jpgs.py +++ b/lantern_scripts/linearize_jpgs.py @@ -5,6 +5,9 @@ from pathlib import Path from tqdm import tqdm import argparse import json +from pathlib import Path +import copy + os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" @@ -22,6 +25,8 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--data_dir', type=str, default='/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/LDR_Nerfacto/pano_jpeg/') parser.add_argument('--out_dir', type=str, default='/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/LDR_Nerfacto/pano/') + parser.add_argument('--do_resize', '-v', action='store_true', help='Enable verbose output') + parser.add_argument("--resolution", type=int, default=256) args = parser.parse_args() well_addr = args.data_dir @@ -33,16 +38,19 @@ if __name__ == "__main__": CORRECTION_CURVE_TYPE = 'gamma' well_images = [] - for path in Path(well_addr).rglob('*.jpg'): + for path in Path(well_addr).glob('*.png'): well_images.append(path) for well_addr in tqdm(well_images): well_addr = str(well_addr) hdr_well_image = cv2.imread(well_addr, cv2.IMREAD_UNCHANGED) + if args.do_resize: + resized_image = cv2.resize(hdr_well_image, (2 * args.resolution, args.resolution), interpolation=cv2.INTER_LANCZOS4) + hdr_well_image = copy.deepcopy(resized_image) final_output = apply_correction((camera_data["b"], camera_data["g"], camera_data["r"]), np.array(hdr_well_image), CORRECTION_CURVE_TYPE) output_addr = well_addr.replace(args.data_dir, args.out_dir) - output_addr = output_addr.replace('.jpg', '.exr') - cv2.imwrite(output_addr, np.float32(final_output) ) - # cv2.imwrite(output_addr, np.float32(final_output*255) ) \ No newline at end of file + # output_addr = output_addr.replace('.png', '.exr') + # cv2.imwrite(output_addr, np.float32(final_output) ) + cv2.imwrite(output_addr, np.float32(final_output*255) ) \ No newline at end of file diff --git a/lantern_scripts/preprocess_for_panoHDRnerf.py b/lantern_scripts/preprocess_for_panoHDRnerf.py new file mode 100644 index 000000000..3eb94f742 --- /dev/null +++ b/lantern_scripts/preprocess_for_panoHDRnerf.py @@ -0,0 +1,82 @@ +import subprocess +import argparse +import os +import shutil + + +if __name__ == '__main__': + argparser = argparse.ArgumentParser() + argparser.add_argument("--input_dir", type=str, required=True, default="/mnt/data/scene/", help="Directory containing everything") + args = argparser.parse_args() + + data_dir = os.path.join(args.input_dir, "data") + left_e1_dir = os.path.join(data_dir, "left_e1") + + left_e1_linearized_dir = os.path.join(data_dir, "left_e1_linearized/") + if os.path.exists(left_e1_linearized_dir): + shutil.rmtree(left_e1_linearized_dir) + os.makedirs(left_e1_linearized_dir) + + left_e1_linearized_resized_dir = os.path.join(data_dir, "left_e1_linearized_resized/") + if os.path.exists(left_e1_linearized_resized_dir): + shutil.rmtree(left_e1_linearized_resized_dir) + + left_e1_ldr2hdr_dir = os.path.join(data_dir, "left_e1_ldr2hdr/") + if os.path.exists(left_e1_ldr2hdr_dir): + shutil.rmtree(left_e1_ldr2hdr_dir) + os.makedirs(left_e1_ldr2hdr_dir) + + left_e1_ldr2hdr_high_res_dir = os.path.join(data_dir, "left_e1_ldr2hdr_high_res/") + if os.path.exists(left_e1_ldr2hdr_high_res_dir): + shutil.rmtree(left_e1_ldr2hdr_high_res_dir) + os.makedirs(left_e1_ldr2hdr_high_res_dir) + left_e1_ldr2hdr_high_res_left_e1_dir = os.path.join(left_e1_ldr2hdr_high_res_dir, "left_e1/") + os.makedirs(left_e1_ldr2hdr_high_res_left_e1_dir) + + print("Linearizing the JPEGs...") + subprocess.run(["conda", "run", "-n", "nerfstudio", "python", "lantern_scripts/linearize_jpgs.py", + "--data_dir", left_e1_dir, + "--out_dir", left_e1_linearized_dir, + ], check=True) + + shutil.copytree(left_e1_linearized_dir, left_e1_linearized_resized_dir) + + print("Resizing target_jpg_resize...") + result = subprocess.run(["find", left_e1_linearized_resized_dir, "-iname", "*.png", "-exec", "convert", "{}", "-verbose", "-resize", "512x256>", "{}", ";"], capture_output=True, text=True) + print(result) + + print("LDR2HDR Network on the data") + subprocess.run(["conda", "run", "-n", "lanet_env_37", "OPENCV_IO_ENABLE_OPENEXR=true", "python", "/mnt/workspace/lantern/PanoHDR-NeRF/LANet/test.py", + "--ckpt", "/mnt/workspace/lantern/PanoHDR-NeRF/LANet/epoch_128.pt", + "--test_dir", left_e1_linearized_resized_dir, + "--output_dir", left_e1_ldr2hdr_dir, + ], check=True) + + print("Combining the LDRs and HDRs...") + subprocess.run(["conda", "run", "-n", "nerfstudio", "python", "lantern_scripts/combine_LDR_HDR.py", + "--ldr_dir", left_e1_linearized_dir, + "--hdr_dir", left_e1_ldr2hdr_dir, + "--output_dir", left_e1_ldr2hdr_high_res_left_e1_dir, + ], check=True) + + sfm_dir = os.path.join(args.input_dir, "sfm") + reconstruction_file = os.path.join(sfm_dir, "reconstruction.json") + + scene_name = str(args.input_dir).split('/')[-2] + output_pano_hdr_nerf_ns = os.path.join(args.input_dir, scene_name + "_pano_hdr_nerf_ns/") + if os.path.exists(output_pano_hdr_nerf_ns): + shutil.rmtree(output_pano_hdr_nerf_ns) + + left_e1_mask_dir = os.path.join(left_e1_dir, "mask/") + target_mask_dir = os.path.join(left_e1_ldr2hdr_high_res_left_e1_dir, "mask/") + if os.path.exists(target_mask_dir): + shutil.rmtree(target_mask_dir) + shutil.copytree(left_e1_mask_dir, target_mask_dir) + + print("Making it ready for nerfstudio...") + subprocess.run(["conda", "run", "-n", "nerfstudio", "ns-process-data", "aligned_pano2plane", + "--data", left_e1_ldr2hdr_high_res_dir, + "--output-dir", output_pano_hdr_nerf_ns, + "--metadata", reconstruction_file, + "--is_metadata_from_openSFM", + ], check=True) \ No newline at end of file diff --git a/nerfstudio/lantern/config.py b/nerfstudio/lantern/config.py index a82eb3c2d..a93b31fb8 100644 --- a/nerfstudio/lantern/config.py +++ b/nerfstudio/lantern/config.py @@ -66,7 +66,12 @@ def Get_PanoHDR_Nerfacto_config(): dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=4096, eval_num_rays_per_batch=4096, - camera_optimizer=None, + # camera_optimizer=None, + camera_optimizer=CameraOptimizerConfig( + mode="SO3xR3", + optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2), + scheduler=ExponentialDecaySchedulerConfig(lr_final=6e-6, max_steps=200000), + ), ), model=PanoHDRNeRFModelConfig(eval_num_rays_per_chunk=1 << 15), ),