diff --git a/lantern_scripts/combine.py b/lantern_scripts/combine.py new file mode 100755 index 000000000..0ae17ed2a --- /dev/null +++ b/lantern_scripts/combine.py @@ -0,0 +1,105 @@ +import os +import numpy as np +import cv2 +from pathlib import Path +from tqdm import tqdm +import argparse +import json + + +FAST_EXPOSURE_CUTOFF = 0.2 # good for real data (0.1 for synthetic) +WELL_EXPOSURE_CUTOFF = 0.98 # good for real data (0.9 for synthetic) + + +def apply_correction(coefs, img, CORRECTION_CURVE_TYPE) -> np.ndarray: + # check that img is in range 0-255 + assert img.dtype == np.uint8 + assert CORRECTION_CURVE_TYPE in ['gamma'] + img = ((img.astype(np.float32) / 255.0) ** np.array(coefs).T) # * 255.0 + # img = ((img.astype(np.float32) / 255.0) ** coefs[None, None, :]) * 255.0 + return img + +def cut_weighting_function(pixels, exposures): + weights = np.zeros(pixels.shape, dtype=np.float32) + + if exposures == WELL_EXPOSURE: + weights = np.clip(-20.0 * (pixels - WELL_EXPOSURE_CUTOFF) + 1, 0.0, 1.0) + else: + weights = np.clip(100.0 * (pixels - FAST_EXPOSURE_CUTOFF), 0.0, 1.0) + + return weights + + +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--well_dir', type=str) + parser.add_argument('--fast_dir', type=str) + parser.add_argument('--out_dir', type=str) + parser.add_argument('--do_linearization', action='store_true') + parser.add_argument('--experiment_location', type=str, required=True) + args = parser.parse_args() + + experiment_setup = args.experiment_location + '/capture_settings.json' + assert os.path.exists(experiment_setup), "!!!! Cannot find the files for exposure setup. !!!" + with open(experiment_setup) as f: + data = json.load(f) + WELL_EXPOSURE = 1.0 + FAST_EXPOSURE = data["right"]["shutter_speed"] / data["left"]["shutter_speed"] + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + well_addr = args.well_dir + + with open(f'linearize-ricoh-z10-wb-3500.json') as f: + camera_data = json.load(f) + CORRECTION_CURVE_TYPE = 'gamma' + + well_images = [] + if args.do_linearization: + for path in Path(well_addr).rglob('*.jpg'): + well_images.append(path) + else: + for path in Path(well_addr).rglob('*.exr'): + well_images.append(path) + + for well_addr in tqdm(well_images): + + well_addr = str(well_addr) + + hdr_well_image = cv2.imread(well_addr, cv2.IMREAD_UNCHANGED) + hdr_fast_image = cv2.imread(well_addr.replace(args.well_dir, args.fast_dir), cv2.IMREAD_UNCHANGED) + + + # hdr_well_image = np.uint8(hdr_well_image * 255.0) + # hdr_fast_image = np.uint8(hdr_fast_image * 255.0) + + if args.do_linearization: + hdr_well_image = apply_correction((camera_data["b"], camera_data["g"], camera_data["r"]), np.array(hdr_well_image), CORRECTION_CURVE_TYPE) + hdr_fast_image = apply_correction((camera_data["b"], camera_data["g"], camera_data["r"]), np.array(hdr_fast_image), CORRECTION_CURVE_TYPE) + + + # well_mask_addr = well_addr.replace('/well_expo/', '/well_mask/') + # well_mask_addr = well_mask_addr.replace('.exr', '.png') + # well_mask = cv2.imread(well_mask_addr) + # well_mask = well_mask.astype(np.float32) / 255.0 + # well_mask[well_mask > 0.1] = 1.0 + + weights_well = cut_weighting_function(hdr_well_image, WELL_EXPOSURE) + weights_fast = cut_weighting_function(hdr_fast_image, FAST_EXPOSURE) + + # fast_mask = cv2.imread(well_mask_addr.replace('/well_mask/', '/fast_mask/')) + # fast_mask = fast_mask.astype(np.float32) / 255.0 + + final_output = weights_well * (hdr_well_image / WELL_EXPOSURE) + weights_fast * (hdr_fast_image / FAST_EXPOSURE) + # final_output = weights_well * (hdr_well_image / WELL_EXPOSURE) + (hdr_fast_image / FAST_EXPOSURE) + # final_output = weights_well * (hdr_well_image / WELL_EXPOSURE) + + output_addr = well_addr.replace(args.well_dir, args.out_dir) + if args.do_linearization: + output_addr = output_addr.replace('jpg', '.exr') + cv2.imwrite(output_addr, np.float32(final_output) ) + # import pdb; pdb.set_trace() \ No newline at end of file diff --git a/lantern_scripts/debevec_bracket_merge.py b/lantern_scripts/debevec_bracket_merge.py new file mode 100644 index 000000000..c3892a397 --- /dev/null +++ b/lantern_scripts/debevec_bracket_merge.py @@ -0,0 +1,123 @@ +import argparse +import numpy as np +import os +from tqdm import tqdm +from PIL import Image +from pathlib import Path +import PIL.ExifTags +import json +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" +import cv2 +import warnings + + +def apply_correction(coefs, img, CORRECTION_CURVE_TYPE) -> np.ndarray: + # check that img is in range 0-255 + assert img.dtype == np.uint8 + assert CORRECTION_CURVE_TYPE in ['gamma'] + img = ((img.astype(np.float32) / 255.0) ** np.array(coefs).T) # * 255.0 + # img = ((img.astype(np.float32) / 255.0) ** coefs[None, None, :]) * 255.0 + return img + + +def hat_weighting_function(pixels, is_lowest = False, is_fastest = False, zmin = 0.0, zmax = 1.0): + threshold = 0.5 * (zmin + zmax) + weights = np.zeros(pixels.shape) + + assert not(is_lowest and is_fastest) + + # mask_lowerbound = zmin <= pixels <= threshold + mask_lowerbound = np.greater_equal(pixels, zmin) & np.less_equal(pixels, threshold) + if not is_lowest: + weights[mask_lowerbound] = 1.0 * (pixels[mask_lowerbound] - zmin) + else: + weights[mask_lowerbound] = 1.0 + + # mask_upperbound = threshold <= pixels <= zmax + mask_upperbound = np.greater_equal(pixels, threshold) & np.less_equal(pixels, zmax) + if not is_fastest: + weights[mask_upperbound] = 1.0 * (zmax - pixels[mask_upperbound]) + else: + weights[mask_upperbound] = 1.0 + + return weights + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--data_dir', type=str, default='/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/GT/HDR_brackets/') + parser.add_argument('--experiment_location', type=str, required=True) + args = parser.parse_args() + + + experiment_setup = args.experiment_location + 'capture_settings.json' + assert os.path.exists(experiment_setup), "!!!! Cannot find the files for exposure setup. !!!" + with open(experiment_setup) as f: + # read shutter_left, shutter_right, iso_left, iso_right + data = json.load(f) + reference_exposure = float(data["left"]["shutter_speed"]) + + + + GT_directories = [x[0] for x in os.walk(args.data_dir)] + GT_directories = list(set(GT_directories) - set([args.data_dir])) + + for data_dir in GT_directories: + test_images = [] + for path in Path(data_dir).rglob('*.JPG'): + test_images.append(path) + + with open(f'linearize-ricoh-z10-wb-3500.json') as f: + camera_data = json.load(f) + CORRECTION_CURVE_TYPE = 'gamma' + + img_0 = np.array(Image.open(test_images[0]) ) + weights_all = np.zeros(img_0.shape) # + np.finfo(np.float32).eps + final_img = np.zeros(img_0.shape) + + fastest_expo = 1000 + lowest_expo = -1000 + + data = {} + + for pano_addr in tqdm(test_images): + img_exif = Image.open(pano_addr) + img = cv2.imread(str(pano_addr)) + exif = { + PIL.ExifTags.TAGS[k]: v + for k, v in img_exif._getexif().items() + if k in PIL.ExifTags.TAGS + } + img_corrected = apply_correction((camera_data["b"], camera_data["g"], camera_data["r"]), np.array(img), CORRECTION_CURVE_TYPE) + # weights = hat_weighting_function(img_corrected) + # weights_all += weights + exposure = float(exif['ExposureTime']) / reference_exposure + data[str(exposure)] = img_corrected + # final_img += (weights * img_corrected) / exposure + + if exposure < fastest_expo: + fastest_expo = exposure + + if lowest_expo < exposure: + lowest_expo = exposure + + for key in data.keys(): + if key == str(lowest_expo): + weights = hat_weighting_function(data[key], is_lowest = True) + elif key == str(fastest_expo): + weights = hat_weighting_function(data[key], is_fastest = True) + else: + weights = hat_weighting_function(data[key]) + weights_all += weights + final_img += (weights * data[key]) / float(key) + + + # import pdb; pdb.set_trace() + # final_img[weights_all == 0] = (1 / fastest_expo) + # weights_all[weights_all == 0] = 1.0 + + + final_img_normal = np.float32(final_img / weights_all) + output_name = data_dir.split('/')[-1] + cv2.imwrite(data_dir + '/'+ output_name + '.exr', final_img_normal ) + cv2.imwrite(data_dir + '/weights_all.exr', np.float32(weights_all) ) \ No newline at end of file diff --git a/lantern_scripts/hdr_blender.py b/lantern_scripts/hdr_blender.py old mode 100644 new mode 100755 index fbd9e839d..d6e0da3a7 --- a/lantern_scripts/hdr_blender.py +++ b/lantern_scripts/hdr_blender.py @@ -12,40 +12,24 @@ import bpy import cycles from mathutils import Euler, Matrix, Vector -# path = 'pano-videos/results/panohdr_log/chess_room/' -path = './' -datasets = ['chess_room','old_class_spot','old_class_neon','small_class'\ - ,'stairway','cafeteria'] -# # out_dirs = ['prof_kitchen/','prof_living_room/','grandma_attic/'\ -# # ,'grandma_guest_room/','chess_room/'\ -# # ,'old_class_spot/','old_class_neon/','small_class/'\ -# # ,'music_room/'] -# # [f'gt/{data}/' for data in datasets] + [f'ldr/{data}/' for data in datasets] -# # [f'blender_multi/gt/{data}/' for data in datasets] + [f'blender_multi/ldr/{data}/' for data in datasets] \ -inp_dirs = ['/mnt/workspace/lantern/renders/lantern-dual-exposure_no_mask/2023-11-01_022121/'] - # [f'results/ft_momo/{data}/mask_pred/' for data in datasets] + \ - # [f'results/1hdr/{data}/mask_pred/' for data in datasets] -out_dirs = ['/mnt/workspace/lantern/renders/lantern-dual-exposure_no_mask/2023-11-01_022121_w_objects/'] - # [f'blender_multi/ft_momo/{data}/mask_pred/' for data in datasets] + \ - # [f'blender_multi/1hdr/{data}/mask_pred/' for data in datasets] +import sys +argv = sys.argv +argv = argv[argv.index("--") + 1:] # get all args after "--" + +inp_dirs = [argv[0]] +out_dirs = [argv[1]] -# inp_dirs = ['final_0/exr/','final_1/exr/','final_2/exr/'] -# out_dirs = ['final_0_render/','final_1_render/','final_2_render/'] -# out_dirs = ['old_class_spot/render_ldr_500000/render/','chess_room/render_ldr_500000/render/', -# 'cafeteria/render_ldr_500000/render/','stairway/render_ldr_500000/render/', -# 'small_class/render_ldr_500000/render/','cafeteria/render_ldr_500000/render/',\ -# 'prof_living_room/render_ldr_500000/render/'] pairs = list(zip(inp_dirs,out_dirs)) scn = bpy.context.scene scn.render.engine = 'CYCLES' scn.render.image_settings.file_format = 'OPEN_EXR' scn.world.use_nodes = True # Enabf_le GPU rendering. -bpy.context.preferences.addons['cycles'].preferences.compute_device_type = 'CUDA' +# bpy.context.preferences.addons['cycles'].preferences.compute_device_type = 'CUDA' bpy.context.preferences.addons['cycles'].preferences.get_devices() for device in bpy.context.preferences.addons['cycles'].preferences.devices: - if "TITAN V" == device.name: + if "TITAN RX" == device.name: device.use = True else: device.use = False @@ -54,23 +38,18 @@ wd = scn.world nt = bpy.data.worlds[wd.name].node_tree for pair in pairs: - inp,out = pair + inp, out = pair inp = inp - out = os.path.join(path,out) os.makedirs(out,exist_ok=True) S = bpy.context.scene envFiles = [f for f in listdir(inp) if isfile(join(inp, f)) and f.endswith('.exr')] - print(envFiles) i = 0 for env_map in envFiles: env_file = join(inp, env_map) bpy.ops.image.open(filepath=env_file, directory=inp, files=[ {"name": env_map, "name": env_map}], show_multiview=False) - output_name = "render_" + env_map.split('.')[0] +'.exr' - # print(output_name) - # bpy.ops.image.save_as(save_as_render=True, copy=True, filepath=join(out_addr, output_name), show_multiview=False, use_multiview=False) - - + output_name = env_map.split('.')[0] +'.exr' + backNode = nt.nodes['Environment Texture'] backNode.image = bpy.data.images.load(env_file) diff --git a/lantern_scripts/ldr_res.py b/lantern_scripts/ldr_res.py new file mode 100644 index 000000000..323ab8635 --- /dev/null +++ b/lantern_scripts/ldr_res.py @@ -0,0 +1,81 @@ +import os +import numpy as np +import cv2 +from pathlib import Path +from tqdm import tqdm +import argparse +import json +import torch +import piq + + +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" + + +def apply_correction(coefs, img, CORRECTION_CURVE_TYPE) -> np.ndarray: + # check that img is in range 0-255 + assert img.dtype == np.uint8 + assert CORRECTION_CURVE_TYPE in ['gamma'] + img = ((img.astype(np.float32) / 255.0) ** np.array(coefs).T) # * 255.0 + # img = ((img.astype(np.float32) / 255.0) ** coefs[None, None, :]) * 255.0 + return img + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--gt_dir', type=str, default='/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/lab_downstairs/GT/pano/') + parser.add_argument('--data_dir', type=str, default='/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/lab_downstairs/HDR-Nerfacto/pano/') + parser.add_argument('--method_name', type=str) + parser.add_argument('--is_exr', action='store_true') + args = parser.parse_args() + well_addr = args.data_dir + + gt_images = [] + for path in Path(args.gt_dir).rglob('*.exr'): + gt_images.append(path) + + pred_images = [] + # if args.is_exr: + for path in Path(args.data_dir).rglob('*.exr'): + pred_images.append(path) + # else: + # for path in Path(args.data_dir).rglob('*.png'): + # pred_images.append(path) + assert len(pred_images) == len(gt_images), "The size of images don't match GT size" + + gts = [] + preds = [] + + # import pdb; pdb.set_trace() + for gt_addr in tqdm(gt_images): + gt_addr = str(gt_addr) + gt_image = cv2.imread(gt_addr, cv2.IMREAD_UNCHANGED) + gt_image[gt_image < 0] = 0.0 + gt_image = np.clip(np.power(gt_image, 1/2.2), 0.0, 1.0) + pred_addr = gt_addr.replace(args.gt_dir, args.data_dir) + # if args.is_exr: + # pred_addr = pred_addr.replace('.png', '.exr') + pred_image = cv2.imread(pred_addr, cv2.IMREAD_UNCHANGED) + pred_image = np.clip(np.power(pred_image, 1/2.2), 0.0, 1.0) + + cv2.imwrite('./test.png', (pred_image*255).astype('uint8')) + cv2.imwrite('./gt.png', (gt_image*255).astype('uint8')) + # else: + # # pred_addr = pred_addr.replace('.png', '.jpg') + # pred_image = cv2.imread(pred_addr, cv2.IMREAD_UNCHANGED) / 255.0 + + if not np.isnan(gt_image).any(): + gt_image_torch = torch.from_numpy(gt_image) + gt_image_torch = gt_image_torch.permute(2, 0, 1) + gts.append(gt_image_torch) + pred_image_torch = torch.from_numpy(pred_image) + pred_image_torch = pred_image_torch.permute(2, 0, 1) + preds.append(pred_image_torch) + + ssim_index = piq.ssim(torch.stack(gts), torch.stack(preds), data_range=1.) + psnr_index = piq.psnr(torch.stack(gts), torch.stack(preds), data_range=1.) + + print("SSIM: ", ssim_index) + print("PSNR: ", psnr_index) + # lpips_index = piq.LPIPS(reduction='none')(torch.stack(gts), torch.stack(preds)) + # print("LPIPS: ", lpips_index) \ No newline at end of file diff --git a/lantern_scripts/linearize_jpgs.py b/lantern_scripts/linearize_jpgs.py new file mode 100644 index 000000000..04927e916 --- /dev/null +++ b/lantern_scripts/linearize_jpgs.py @@ -0,0 +1,45 @@ +import os +import numpy as np +import cv2 +from pathlib import Path +from tqdm import tqdm +import argparse +import json + +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" + + +def apply_correction(coefs, img, CORRECTION_CURVE_TYPE) -> np.ndarray: + # check that img is in range 0-255 + assert img.dtype == np.uint8 + assert CORRECTION_CURVE_TYPE in ['gamma'] + img = ((img.astype(np.float32) / 255.0) ** np.array(coefs).T) # * 255.0 + # img = ((img.astype(np.float32) / 255.0) ** coefs[None, None, :]) * 255.0 + return img + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--data_dir', type=str, default='/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/LDR_Nerfacto/pano_jpeg/') + parser.add_argument('--out_dir', type=str, default='/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/LDR_Nerfacto/pano/') + args = parser.parse_args() + well_addr = args.data_dir + + with open(f'./linearize-ricoh-z10-wb-3500.json') as f: + camera_data = json.load(f) + CORRECTION_CURVE_TYPE = 'gamma' + + well_images = [] + for path in Path(well_addr).rglob('*.jpg'): + well_images.append(path) + + for well_addr in tqdm(well_images): + + well_addr = str(well_addr) + hdr_well_image = cv2.imread(well_addr, cv2.IMREAD_UNCHANGED) + final_output = apply_correction((camera_data["b"], camera_data["g"], camera_data["r"]), np.array(hdr_well_image), CORRECTION_CURVE_TYPE) + + output_addr = well_addr.replace(args.data_dir, args.out_dir) + output_addr = output_addr.replace('.jpg', '.exr') + cv2.imwrite(output_addr, np.float32(final_output) ) + # cv2.imwrite(output_addr, np.float32(final_output*255) ) \ No newline at end of file diff --git a/lantern_scripts/rectify_pano.py b/lantern_scripts/rectify_pano.py new file mode 100644 index 000000000..da85afeba --- /dev/null +++ b/lantern_scripts/rectify_pano.py @@ -0,0 +1,52 @@ +import argparse +import numpy as np +import os +import cv2 +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" + +from imageio import imread, imsave +from envmap import EnvironmentMap, rotation_matrix +import math +from tqdm import tqdm +from pathlib import Path + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--data_dir', type=str, default='/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/Henry_Gagnon_red_and_blue/PanoHDR-NeRF/left_e1_ldr2hdr/') + parser.add_argument('--out_dir', type=str, default='/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/Henry_Gagnon_red_and_blue/PanoHDR-NeRF/left_e1_ldr2hdr_unrectify/') + parser.add_argument('--unrectify', action='store_true') + args = parser.parse_args() + + test_images = [] + if args.unrectify: + for path in Path(args.data_dir).rglob('*.exr'): + test_images.append(path) + else: + for path in Path(args.data_dir).rglob('*.png'): + test_images.append(path) + + RicohPitch = math.radians(-85.4) + RicohAzimuth = math.radians(132.03) + RicohRoll = math.radians(0) + + for pano_addr in tqdm(test_images): + pano_addr = str(pano_addr) + e = EnvironmentMap(pano_addr, 'latlong') + + dcm = rotation_matrix(azimuth=0, elevation=RicohAzimuth, roll=RicohPitch) + from numpy.linalg import inv + if args.unrectify: + dcm = inv(dcm) + e.rotate(dcm) + + out_dir = pano_addr.replace(args.data_dir, args.out_dir) + + if args.unrectify: + rectified = np.float32(e.data) + rectified = cv2.cvtColor(rectified, cv2.COLOR_RGB2BGR) + cv2.imwrite(out_dir, rectified) + else: + rectified = np.clip(255.*e.data, 0, 255).astype('uint8') + imsave(out_dir, rectified, quality=100) + \ No newline at end of file diff --git a/lantern_scripts/render_all.py b/lantern_scripts/render_all.py new file mode 100644 index 000000000..60658a846 --- /dev/null +++ b/lantern_scripts/render_all.py @@ -0,0 +1,70 @@ +""" +Script to render IBL renders +Usage: blender -P hdr_blender.py -b +""" +import os +import random +from math import radians +from os import listdir +from os.path import isfile, join + +import bpy +import cycles +from mathutils import Euler, Matrix, Vector + + +RENDER_OUT = '/render/' +path = './' + +data_dir = '/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/' +# method_directories = [x[0] for x in os.walk(data_dir)] +method_directories = os.listdir(data_dir) +method_directories = list(set(method_directories) - set([data_dir, '.DS_Store', 'capture_settings.json'])) + +inp_dirs = [] +out_dirs = [] +for method_dir in method_directories: + input_dir = data_dir + method_dir + '/pano/' + inp_dirs.append(input_dir) + output_dir = data_dir + method_dir + RENDER_OUT + out_dirs.append(output_dir) + + +pairs = list(zip(inp_dirs,out_dirs)) +scn = bpy.context.scene +scn.render.engine = 'CYCLES' +scn.render.image_settings.file_format = 'OPEN_EXR' +scn.world.use_nodes = True +# Enabf_le GPU rendering. +# bpy.context.preferences.addons['cycles'].preferences.compute_device_type = 'CUDA' +bpy.context.preferences.addons['cycles'].preferences.get_devices() +for device in bpy.context.preferences.addons['cycles'].preferences.devices: + if "TITAN V" == device.name: + device.use = True + else: + device.use = False +# select world node tree +wd = scn.world +nt = bpy.data.worlds[wd.name].node_tree + +for pair in pairs: + inp,out = pair + inp = inp + out = os.path.join(path,out) + os.makedirs(out,exist_ok=True) + S = bpy.context.scene + envFiles = [f for f in listdir(inp) if isfile(join(inp, f)) and f.endswith('.exr')] + print(envFiles) + i = 0 + for env_map in envFiles: + env_file = join(inp, env_map) + bpy.ops.image.open(filepath=env_file, directory=inp, files=[ + {"name": env_map, "name": env_map}], show_multiview=False) + output_name = env_map.split('.')[0] +'.exr' + + backNode = nt.nodes['Environment Texture'] + backNode.image = bpy.data.images.load(env_file) + + bpy.context.scene.render.filepath = os.path.join(out, output_name) + print("Output",os.path.join(out, output_name)) + bpy.ops.render.render(write_still=True) diff --git a/lantern_scripts/res_table_1.py b/lantern_scripts/res_table_1.py new file mode 100644 index 000000000..ec5bd827e --- /dev/null +++ b/lantern_scripts/res_table_1.py @@ -0,0 +1,185 @@ +from ezexr import imread +import os +import numpy as np +from tqdm import tqdm +from math import log10 +import json +import cv2 +import imageio + + +def wrmse(gt, est, mask): + if mask is None: + gt = gt.flatten() + est = est.flatten() + else: + gt = gt[mask].flatten() + est = est[mask].flatten() + error = np.sqrt(np.mean(np.power(gt - est, 2))) + + return error + +def si_wrmse(gt, est, mask): + if mask is None: + gt_c = gt.flatten() + est_c = est.flatten() + else: + gt_c = gt[mask].flatten() + est_c = est[mask].flatten() + alpha = (np.dot(np.transpose(gt_c), est_c)) / (np.dot(np.transpose(est_c), est_c)) + error = wrmse(gt, est * alpha, mask) + + return error + +def tonemap(img, gamma=2.4): + """Apply gamma, then clip between 0 and 1, finally convert to uint8 [0,255]""" + return (np.clip(np.power(img,1/gamma), 0.0, 1.0)*255).astype('uint8') + + +def angular_error(gt_render, pred_render, mask=None): + # The error need to be computed with the normalized rgb image. + # Normalized RGB is r = R / (R+G+B), g = G / (R+G+B), b = B / (R+G+B) + # The angular distance is the distance between pixel 1 and pixel 2. + # It's computed with cos^-1(p1·p2 / ||p1||*||p2||) + gt_norm = np.empty((gt_render.shape)) + pred_norm = np.empty(pred_render.shape) + + for i in range(3): + gt_norm[:,:,i] = gt_render[:,:,i] / np.sum(gt_render, axis=2, keepdims=True)[:,:,0] + pred_norm[:,:,i] = pred_render[:,:,i] / (np.sum(pred_render, axis=2, keepdims=True)[:,:,0] + 1e-8) + + angular_error_arr = np.arccos( np.sum(gt_norm*pred_norm, axis=2, keepdims=True)[:,:,0] / + ((np.sqrt(np.sum(gt_norm*gt_norm, axis=2, keepdims=True)[:,:,0])*np.sqrt(np.sum(pred_norm*pred_norm, axis=2, keepdims=True)[:,:,0]))) ) + + if mask is not None: + angular_error_arr = angular_error_arr[mask[:,:,0]] + else: + angular_error_arr = angular_error_arr.flatten() + angular_error_arr = angular_error_arr[~np.isnan(angular_error_arr)] + mean = np.mean(angular_error_arr) + # convert to degree + mean = mean * 180 / np.pi + return mean + + +def psnr(original, compressed): + original_tone_mapped = tonemap(original) + compressed_tone_mapped = tonemap(compressed) + + cv2.imwrite('./out.png', cv2.cvtColor(compressed_tone_mapped, cv2.COLOR_RGB2BGR)) + cv2.imwrite('./gt.png', cv2.cvtColor(original_tone_mapped, cv2.COLOR_RGB2BGR)) + + mse = wrmse(original_tone_mapped, compressed_tone_mapped, None) + if(mse == 0): # MSE is zero means no noise is present in the signal . + # Therefore PSNR have no importance. + return 100 + max_pixel = 255 #max(original.max(), compressed.max()) + psnr = 20 * log10(max_pixel / mse) + return psnr + + +def remove_nan_and_sort(x): + x = np.array(x) + x = x[~np.isnan(x)] + x = np.sort(x) + return x + + +results_dataset_roots = [ + '/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/LDR-Nerfacto/render/', + '/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/linearization_then_nerf/render/', + '/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/nerf_then_linearization/render/', + # '/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/HDR-Nerfacto/render/', + # '/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/PanoHDR-NeRF/render/' + ] +# results_dataset_roots = '/home/karimida/evaluation/final_results/Henrique_data/my_renderings_of_Henrique_2' + +dataset_names = [ + 'LDR_nerfacto', + 'linearization_then_NeRF', + 'NeRF_then_linearization', + # 'HDR-Nerfacto', + # 'PanoHDRNeRF' + ] + +mse = {} +si_mse = {} +angular = {} +psnr_error = {} +for key in dataset_names: + mse[key] = [] + si_mse[key] = [] + angular[key] = [] + psnr_error[key] = [] + +# Ground truth for the indoor +# gt_dataset_root = '/home/karimida/evaluation/redoing_table_one/hdr_env_maps/gt_render_diffuse/' + +# Ground truth for the outdoor +gt_dataset_root = '/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/GT/render/' +output_folder = '/Users/momo/Desktop' + +gt_dataset_files = sorted([os.path.join(gt_dataset_root, f) for f in os.listdir(gt_dataset_root) if f.endswith('.exr')]) + +mask_dataset_root = '/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/GT/render_mask/' + +# create output folder output_folder, 'metrics' +os.system('mkdir -p ' + output_folder + '/metrics') + +for results_dataset_root, dataset_name in zip(results_dataset_roots, dataset_names): + + for gt_dataset_file in tqdm(gt_dataset_files): + gt_dataset_img_exr = imread(gt_dataset_file)[:, :, :3].astype(np.float32) + + result_dataset_file = gt_dataset_file.replace(gt_dataset_root, results_dataset_root) + result_dataset_img_exr = imread(result_dataset_file)[:, :, :3].astype(np.float32) + + mask_file = gt_dataset_file.replace(gt_dataset_root, mask_dataset_root) + mask_file = mask_file.replace('.exr', '.png') + mask = np.float32(imageio.imread(mask_file)) / 255.0 + mask = mask == 1 + mask = np.repeat(mask[:, :, np.newaxis], 3, axis=2) + gt_dataset_img_exr_for_PSNR = gt_dataset_img_exr[mask] + result_dataset_img_exr_for_PSNR = result_dataset_img_exr[mask] + + mse_result = wrmse(gt_dataset_img_exr, result_dataset_img_exr, mask) + scale_invariant_mse_result = si_wrmse(gt_dataset_img_exr, result_dataset_img_exr, mask) + angular_result = angular_error(gt_dataset_img_exr, result_dataset_img_exr, mask) + psnr_result = psnr(gt_dataset_img_exr_for_PSNR, result_dataset_img_exr_for_PSNR) + mse[dataset_name].append(mse_result) + si_mse[dataset_name].append(scale_invariant_mse_result) + angular[dataset_name].append(angular_result) + psnr_error[dataset_name].append(psnr_result) + si_mse[dataset_name].append(scale_invariant_mse_result) + + + + # mse = {key: remove_nan_and_sort(mse[key]) for key in mse} + mse[dataset_name] = remove_nan_and_sort(mse[dataset_name]) + + # mse_ours = np.percentile(mse[dataset_name], q=50) + # si_mse_ours = np.percentile(si_mse[dataset_name], q=50) + # angular_ours = np.percentile(angular[dataset_name], q=50) + # psnr_ours = np.percentile(psnr_error[dataset_name], q=50) + + mse_ours = np.mean(mse[dataset_name]) + si_mse_ours = np.mean(si_mse[dataset_name]) + angular_ours = np.mean(angular[dataset_name]) + psnr_ours = np.mean(psnr_error[dataset_name]) + + # print(dataset_name + ', si_RMSE: {}'.format(si_mse_ours)) + # print(dataset_name + ', RMSE: {}'.format(mse_ours)) + # print(dataset_name + ', angular: {}'.format(angular_ours)) + # print(dataset_name + ', PSNR: {}'.format(psnr_ours)) + + output_dict = {} + output_dict['model_name'] = dataset_name + output_dict['si_RMSE'] = str(si_mse_ours) + output_dict['RMSE'] = str(mse_ours) + output_dict['angular'] = str(angular_ours) + output_dict['PSNR'] = str(psnr_ours) + + output_addr = output_folder + '/metrics/' + dataset_name + '_results.json' + # print("output_addr: ", output_addr) + with open(output_addr, 'w') as f: + json.dump(output_dict, f, indent=4) \ No newline at end of file diff --git a/lantern_scripts/scene_new.blend b/lantern_scripts/scene_new.blend new file mode 100644 index 000000000..6e0c19070 Binary files /dev/null and b/lantern_scripts/scene_new.blend differ diff --git a/lantern_scripts/show_methods.py b/lantern_scripts/show_methods.py new file mode 100644 index 000000000..4444d31c4 --- /dev/null +++ b/lantern_scripts/show_methods.py @@ -0,0 +1,61 @@ +import os +import numpy as np +import cv2 +from pathlib import Path +from tqdm import tqdm +import argparse +import json +import numpy as np +import matplotlib.pyplot as plt + + +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" + +def tonemap(img, gamma=2.2): + """Apply gamma, then clip between 0 and 1, finally convert to uint8 [0,255]""" + return (np.clip(np.power(img,1/gamma), 0.0, 1.0)*255).astype('uint8') + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--data_dir', type=str, default='/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/Henry_Gagnon_red_and_blue/') + args = parser.parse_args() + + # methods = ['LDR-Nerfacto', 'linearization_then_nerf', 'nerf_then_linearization', 'PanoHDR-NeRF', 'HDR-Nerfacto'] + methods = ['LDR-Nerfacto', 'PanoHDR-NeRF', 'HDR-Nerfacto', 'linearization_then_nerf', 'nerf_then_linearization'] + # methods = ['LDR-Nerfacto', 'linearization_then_nerf', 'nerf_then_linearization'] + + + gt_folder = args.data_dir + 'GT' + '/render/' + gt_images = [] + for path in Path(gt_folder).rglob('*.exr'): + gt_images.append(str(path)) + + fig = plt.figure(figsize=(8, 8)) + columns = len(methods) + 1 + rows = len(gt_images) + + i = 1 + for gt_addr in gt_images: + gt_img = cv2.imread(gt_addr, cv2.IMREAD_UNCHANGED) + gt_img = cv2.cvtColor(gt_img, cv2.COLOR_BGR2RGB) + gt_img = tonemap(gt_img) + for method in tqdm(methods): + method_folder = args.data_dir + method + '/render/' + img_addr = gt_addr.replace(gt_folder, method_folder) + inp_img = cv2.imread(img_addr, cv2.IMREAD_UNCHANGED) + inp_img = cv2.cvtColor(inp_img, cv2.COLOR_BGR2RGB) + inp_img = tonemap(inp_img) + fig.add_subplot(rows, columns, i) + i += 1 + plt.axis('off') + plt.imshow(inp_img) + plt.title(method) + fig.add_subplot(rows, columns, i) + i += 1 + plt.imshow(gt_img) + plt.axis('off') + plt.title('GT') + + plt.show() \ No newline at end of file diff --git a/lantern_scripts/un_gamma.py b/lantern_scripts/un_gamma.py new file mode 100644 index 000000000..9253534ef --- /dev/null +++ b/lantern_scripts/un_gamma.py @@ -0,0 +1,34 @@ +import argparse +import numpy as np +import os +import cv2 +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" + +from imageio import imread, imsave +from envmap import EnvironmentMap, rotation_matrix +import math +from tqdm import tqdm +from pathlib import Path + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--data_dir', type=str, default='/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/Henry_Gagnon_red_and_blue/PanoHDR-NeRF/pano_gamma_tone_mapped/') + parser.add_argument('--out_dir', type=str, default='/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/Henry_Gagnon_red_and_blue/PanoHDR-NeRF/pano/') + args = parser.parse_args() + + test_images = [] + for path in Path(args.data_dir).rglob('*.exr'): + test_images.append(path) + + for pano_addr in tqdm(test_images): + pano_addr = str(pano_addr) + e = EnvironmentMap(pano_addr, 'latlong') + + data = e.data**2.2 + + out_dir = pano_addr.replace(args.data_dir, args.out_dir) + + data = np.float32(data) + data = cv2.cvtColor(data, cv2.COLOR_RGB2BGR) + cv2.imwrite(out_dir, data) \ No newline at end of file diff --git a/lantern_scripts/un_mu_law.py b/lantern_scripts/un_mu_law.py new file mode 100755 index 000000000..d907f75ca --- /dev/null +++ b/lantern_scripts/un_mu_law.py @@ -0,0 +1,42 @@ +import argparse +import numpy as np +import os +import cv2 +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" + +from imageio import imread, imsave +from envmap import EnvironmentMap, rotation_matrix +import math +from tqdm import tqdm +from pathlib import Path + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--data_dir', type=str, default='/Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/linearization_then_nerf/fast_compressed_mu_law/') + parser.add_argument('--out_dir', type=str, default='//Users/momo/Desktop/Pandora-HDR_NeRF/final_results/meeting_room/linearization_then_nerf/fast_expo/') + args = parser.parse_args() + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + test_images = [] + for path in Path(args.data_dir).rglob('*.exr'): + test_images.append(path) + + for pano_addr in tqdm(test_images): + pano_addr = str(pano_addr) + e = EnvironmentMap(pano_addr, 'latlong') + + data = e.data**2.2 + + u = 5000. + img_uncompress = np.exp(e.data * np.log(u+1.)) - 1. + img_uncompress /= u + data = img_uncompress + + out_dir = pano_addr.replace(args.data_dir, args.out_dir) + + data = np.float32(data) + data = cv2.cvtColor(data, cv2.COLOR_RGB2BGR) + cv2.imwrite(out_dir, data) \ No newline at end of file diff --git a/nerf_then_linearization_evaluation.sh b/nerf_then_linearization_evaluation.sh new file mode 100755 index 000000000..5f7ddf6ae --- /dev/null +++ b/nerf_then_linearization_evaluation.sh @@ -0,0 +1,28 @@ +#!/bin/bash +echo "Evaluation for linearization then nerf pipeline" +dataset_dir=/mnt/data/nerfstudio_ds/real_data/Henry_Gagnon_blue_et_rouge/Henry_Gagnon_blue_et_rouge_ns/ +opensfm_dir=/mnt/data/nerfstudio_ds/real_data/Henry_Gagnon_blue_et_rouge/Henry_Gagnon_blue_et_rouge_openSFM + +checkpoint_dir=outputs/Henry_Gagnon_blue_et_rouge_ns/lantern-nerfacto/2024-05-16_015020/ +output_dir=renders/Henry_Gagnon_blue_et_rouge_ns-nerfacto/nerf_then_linearization/test_w_camera_optimization_2024-05-16_015020 + +opensfm_file=$opensfm_dir/reconstruction.json + +echo "Creating the test set" +ns-process-data lantern-GT-HDR --data $dataset_dir --output-dir $dataset_dir --metadata $opensfm_file --checkpoint $checkpoint_dir + +echo "Rendering for the well exposed" +ns-render camera-path --load-config $checkpoint_dir/config.yml --camera-path-filename $dataset_dir/GT_transforms.json --output-path $output_dir/well-expo/ --output-format images + +echo "Rendering for the fast exposure" +ns-render camera-path --load-config $checkpoint_dir/config.yml --camera-path-filename $dataset_dir/GT_transforms.json --output-path $output_dir/fast-expo/ --output-format images --rendered_output_names rgb_fast + +#conda init bash +conda activate everlight_new + +echo "Combine two exposures" +python lantern_scripts/combine.py --well_dir $output_dir/well-expo/ --fast_dir $output_dir/fast-expo/ --out_dir $output_dir/pano/ --do_linearization --experiment_location $root_dir + +conda activate nerfstudio-HDR + +blender --background lantern_scripts/scene_new.blend -P lantern_scripts/hdr_blender.py -- $output_dir/pano/ $output_dir/render/ \ No newline at end of file diff --git a/nerfstudio/data/pixel_samplers.py b/nerfstudio/data/pixel_samplers.py index b00d29d8a..2e1d3fed4 100644 --- a/nerfstudio/data/pixel_samplers.py +++ b/nerfstudio/data/pixel_samplers.py @@ -267,7 +267,7 @@ class PixelSampler: chan, y, x = (i.flatten() for i in torch.split(indices, 1, dim=-1)) chan, y, x = chan.cpu(), y.cpu(), x.cpu() collated_batch = { - key: value[chan, y, x] for key, value in batch.items() if key != "image_idx" and key != "exposure" and value is not None + key: value[chan, y, x] for key, value in batch.items() if key != "image_idx" and key != "exposure" and key != "image_filename" and value is not None } assert collated_batch["image"].shape[0] == num_rays_per_batch @@ -375,7 +375,7 @@ class PixelSampler: if not(self.well_image_batch and self.fast_image_batch): well_ones = (image_batch["exposure"] == 1.0).cpu() fast_ones = ~well_ones - for key in image_batch.keys(): + for key in image_batch.keys() - ['image_filename']: self.well_image_batch[key] = image_batch[key][well_ones] self.fast_image_batch[key] = image_batch[key][fast_ones] @@ -392,14 +392,23 @@ class PixelSampler: self.fast_image_batch, int(0.0 * self.num_rays_per_batch), keep_full_image=self.config.keep_full_image, is_fast_expo=True, only_light_sources=True ) else: + # well_pixel_batch = self.collate_image_dataset_batch( + # self.well_image_batch, int(0.2 * self.num_rays_per_batch), keep_full_image=self.config.keep_full_image + # ) + # fast_pixel_batch = self.collate_image_dataset_batch( + # self.fast_image_batch, int(0.3 * self.num_rays_per_batch), keep_full_image=self.config.keep_full_image, is_fast_expo=True + # ) + # light_pixel_batch = self.collate_image_dataset_batch( + # self.fast_image_batch, int(0.5 * self.num_rays_per_batch), keep_full_image=self.config.keep_full_image, is_fast_expo=True, only_light_sources=True + # ) well_pixel_batch = self.collate_image_dataset_batch( - self.well_image_batch, int(0.2 * self.num_rays_per_batch), keep_full_image=self.config.keep_full_image + self.well_image_batch, int(0.0 * self.num_rays_per_batch), keep_full_image=self.config.keep_full_image ) fast_pixel_batch = self.collate_image_dataset_batch( self.fast_image_batch, int(0.3 * self.num_rays_per_batch), keep_full_image=self.config.keep_full_image, is_fast_expo=True ) light_pixel_batch = self.collate_image_dataset_batch( - self.fast_image_batch, int(0.5 * self.num_rays_per_batch), keep_full_image=self.config.keep_full_image, is_fast_expo=True, only_light_sources=True + self.fast_image_batch, int(0.7 * self.num_rays_per_batch), keep_full_image=self.config.keep_full_image, is_fast_expo=True, only_light_sources=True ) pixel_batch = well_pixel_batch #.copy() diff --git a/nerfstudio/fields/nerfacto_field.py b/nerfstudio/fields/nerfacto_field.py index 580d15f7a..7dc2ddcd1 100644 --- a/nerfstudio/fields/nerfacto_field.py +++ b/nerfstudio/fields/nerfacto_field.py @@ -252,8 +252,8 @@ class NerfactoField(Field): selector = ((positions > 0.0) & (positions < 1.0)).all(dim=-1) positions = positions * selector[..., None] self._sample_locations = positions - # if not self._sample_locations.requires_grad: - # self._sample_locations.requires_grad = True + if not self._sample_locations.requires_grad: + self._sample_locations.requires_grad = True # import pdb; pdb.set_trace() positions_flat = positions.view(-1, 3) h = self.mlp_base(positions_flat).view(*ray_samples.frustums.shape, -1) diff --git a/nerfstudio/lantern/config.py b/nerfstudio/lantern/config.py index d9356e353..0aacdecdd 100644 --- a/nerfstudio/lantern/config.py +++ b/nerfstudio/lantern/config.py @@ -30,12 +30,12 @@ def Get_lantern_config(): dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=8192, eval_num_rays_per_batch=8192, - # camera_optimizer=CameraOptimizerConfig( - # mode="SO3xR3", - # optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2), - # scheduler=ExponentialDecaySchedulerConfig(lr_final=6e-6, max_steps=200000), - # ), - camera_optimizer=None, + camera_optimizer=CameraOptimizerConfig( + mode="SO3xR3", + optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2), + scheduler=ExponentialDecaySchedulerConfig(lr_final=6e-6, max_steps=200000), + ), + # camera_optimizer=None, ), model=LanternModelConfig(eval_num_rays_per_chunk=1 << 15), ), @@ -66,11 +66,6 @@ def Get_PanoHDR_Nerfacto_config(): dataparser=NerfstudioDataParserConfig(), train_num_rays_per_batch=4096, eval_num_rays_per_batch=4096, - # camera_optimizer=CameraOptimizerConfig( - # mode="SO3xR3", - # optimizer=AdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-2), - # scheduler=ExponentialDecaySchedulerConfig(lr_final=6e-6, max_steps=200000), - # ), camera_optimizer=None, ), model=PanoHDRNeRFModelConfig(eval_num_rays_per_chunk=1 << 15), diff --git a/nerfstudio/lantern/field.py b/nerfstudio/lantern/field.py index 9dbfe31b8..aee80d9dd 100644 --- a/nerfstudio/lantern/field.py +++ b/nerfstudio/lantern/field.py @@ -68,6 +68,7 @@ class LanternNerfactoField(NerfactoField): implementation: Literal["tcnn", "torch"] = "tcnn", use_appearance_embedding: bool = True, predicts_validity: bool = True, + second_step: bool = False, ) -> None: super().__init__(aabb, num_images, num_layers, hidden_dim, geo_feat_dim, num_levels, base_res, @@ -124,8 +125,19 @@ class LanternNerfactoField(NerfactoField): out_activation=nn.Sigmoid(), implementation=implementation, ) + self.second_step = second_step + def get_density(self, ray_samples: RaySamples) -> Tuple[Tensor, Tensor]: + """Computes and returns the densities.""" + if self.second_step: + with torch.no_grad(): + density, base_mlp_out = super().get_density(ray_samples) + else: + density, base_mlp_out = super().get_density(ray_samples) + return density, base_mlp_out + + def get_outputs( self, ray_samples: RaySamples, density_embedding: Optional[Tensor] = None ) -> Dict[FieldHeadNames, Tensor]: diff --git a/nerfstudio/lantern/model.py b/nerfstudio/lantern/model.py index 516ef3ee8..f522bc02f 100644 --- a/nerfstudio/lantern/model.py +++ b/nerfstudio/lantern/model.py @@ -97,6 +97,7 @@ class LanternModel(NerfactoModel): use_average_appearance_embedding=self.config.use_average_appearance_embedding, appearance_embedding_dim=self.config.appearance_embed_dim, implementation=self.config.implementation, + second_step=(self.config.lantern_steps == 2) ) self.density_fns = [] @@ -234,7 +235,7 @@ class LanternModel(NerfactoModel): ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: gt_rgb = batch["image"].to(self.device) - if batch["exposure"] == 1.0: + if batch["exposure"] != 1.0: predicted_rgb = outputs["rgb"] # Blended with background (black if random background) else: # if self.config.lantern_steps == 1: @@ -256,6 +257,8 @@ class LanternModel(NerfactoModel): print("Exposure: ", batch['exposure']) combined_rgb = torch.cat([gt_rgb, predicted_rgb], dim=1) + # print("combined_rgb: ", combined_rgb.shape, "gt_rgb: ", gt_rgb.shape, "predicted_rgb: ", predicted_rgb.shape) + combined_rgb = predicted_rgb # torch.cat([gt_rgb, predicted_rgb], dim=1) combined_acc = torch.cat([acc], dim=1) combined_depth = torch.cat([depth], dim=1) @@ -280,6 +283,7 @@ class LanternModel(NerfactoModel): metrics_dict["lpips"] = float(lpips) images_dict = {"img": combined_rgb, "accumulation": combined_acc, "depth": combined_depth} + # images_dict = {"img": predicted_rgb, "accumulation": combined_acc, "depth": combined_depth} for i in range(self.config.num_proposal_iterations): key = f"prop_depth_{i}"