From 8ee91791d952b1720f257d94605a0b6eb4bb5bed Mon Sep 17 00:00:00 2001 From: frfoc1 Date: Sun, 19 May 2024 17:08:41 -0400 Subject: [PATCH] Deadline results --- lantern_scripts/combine.py | 106 ++++++++++++++++++ linearize-ricoh-z10-wb-3500.json | 2 +- linearize-ricoh-z10-wb-5000.json | 1 - video_rendering_code/hdr_blender.py | 58 ++++++++++ video_rendering_code/render_video.sh | 160 +++++++++++++++++++++++++++ 5 files changed, 325 insertions(+), 2 deletions(-) create mode 100644 lantern_scripts/combine.py delete mode 100644 linearize-ricoh-z10-wb-5000.json create mode 100644 video_rendering_code/hdr_blender.py create mode 100644 video_rendering_code/render_video.sh diff --git a/lantern_scripts/combine.py b/lantern_scripts/combine.py new file mode 100644 index 000000000..7c2035113 --- /dev/null +++ b/lantern_scripts/combine.py @@ -0,0 +1,106 @@ +import os +import numpy as np +import cv2 +from pathlib import Path +from tqdm import tqdm +import argparse +import json + + +FAST_EXPOSURE_CUTOFF = 0.1 # good for real data (0.1 for synthetic) +WELL_EXPOSURE_CUTOFF = 0.98 # good for real data (0.9 for synthetic) + + +def apply_correction(coefs, img, CORRECTION_CURVE_TYPE) -> np.ndarray: + # check that img is in range 0-255 + assert img.dtype == np.uint8 + assert CORRECTION_CURVE_TYPE in ['gamma'] + img = ((img.astype(np.float32) / 255.0) ** np.array(coefs).T) # * 255.0 + # img = ((img.astype(np.float32) / 255.0) ** coefs[None, None, :]) * 255.0 + return img + +def cut_weighting_function(pixels, exposures): + weights = np.zeros(pixels.shape, dtype=np.float32) + + if exposures == WELL_EXPOSURE: + weights = np.clip(-20.0 * (pixels - WELL_EXPOSURE_CUTOFF) + 1, 0.0, 1.0) + else: + weights = np.clip(100.0 * (pixels - FAST_EXPOSURE_CUTOFF), 0.0, 1.0) + + return weights + + +os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--well_dir', type=str) + parser.add_argument('--fast_dir', type=str) + parser.add_argument('--out_dir', type=str) + parser.add_argument('--do_linearization', action='store_true') + parser.add_argument('--experiment_location', type=str, required=True) + args = parser.parse_args() + + experiment_setup = args.experiment_location + '/capture_settings.json' + assert os.path.exists(experiment_setup), "!!!! Cannot find the files for exposure setup. !!!" + with open(experiment_setup) as f: + data = json.load(f) + WELL_EXPOSURE = 1.0 + FAST_EXPOSURE = data["right"]["shutter_speed"] / data["left"]["shutter_speed"] + + if not os.path.exists(args.out_dir): + os.makedirs(args.out_dir) + + well_addr = args.well_dir + + with open(f'linearize-ricoh-z10-wb-3500.json') as f: + camera_data = json.load(f) + CORRECTION_CURVE_TYPE = 'gamma' + + well_images = [] + if args.do_linearization: + for path in Path(well_addr).rglob('*.jpg'): + well_images.append(path) + else: + for path in Path(well_addr).rglob('*.exr'): + well_images.append(path) + + for well_addr in tqdm(well_images): + + well_addr = str(well_addr) + + hdr_well_image = cv2.imread(well_addr, cv2.IMREAD_UNCHANGED) + hdr_fast_image = cv2.imread(well_addr.replace(args.well_dir, args.fast_dir), cv2.IMREAD_UNCHANGED) + + + # hdr_well_image = np.uint8(hdr_well_image * 255.0) + # hdr_fast_image = np.uint8(hdr_fast_image * 255.0) + + if args.do_linearization: + hdr_well_image = apply_correction((camera_data["b"], camera_data["g"], camera_data["r"]), np.array(hdr_well_image), CORRECTION_CURVE_TYPE) + hdr_fast_image = apply_correction((camera_data["b"], camera_data["g"], camera_data["r"]), np.array(hdr_fast_image), CORRECTION_CURVE_TYPE) + + + # well_mask_addr = well_addr.replace('/well_expo/', '/well_mask/') + # well_mask_addr = well_mask_addr.replace('.exr', '.png') + # well_mask = cv2.imread(well_mask_addr) + # well_mask = well_mask.astype(np.float32) / 255.0 + # well_mask[well_mask > 0.1] = 1.0 + + weights_well = cut_weighting_function(hdr_well_image, WELL_EXPOSURE) + weights_fast = cut_weighting_function(hdr_fast_image, FAST_EXPOSURE) + + # fast_mask = cv2.imread(well_mask_addr.replace('/well_mask/', '/fast_mask/')) + # fast_mask = fast_mask.astype(np.float32) / 255.0 + + # final_output = weights_well * (hdr_well_image / WELL_EXPOSURE) + weights_fast * (hdr_fast_image / FAST_EXPOSURE) + # final_output = final_output / (weights_well+weights_well) + final_output = weights_well * (hdr_well_image / WELL_EXPOSURE) + (hdr_fast_image / FAST_EXPOSURE) + # final_output = weights_well * (hdr_well_image / WELL_EXPOSURE) + + output_addr = well_addr.replace(args.well_dir, args.out_dir) + if args.do_linearization: + output_addr = output_addr.replace('jpg', '.exr') + cv2.imwrite(output_addr, np.float32(final_output) ) + # import pdb; pdb.set_trace() \ No newline at end of file diff --git a/linearize-ricoh-z10-wb-3500.json b/linearize-ricoh-z10-wb-3500.json index 6ff6171b1..112314c91 100644 --- a/linearize-ricoh-z10-wb-3500.json +++ b/linearize-ricoh-z10-wb-3500.json @@ -1 +1 @@ -{"B": [54.09578323364258, 0.2747989296913147, 0.0019784104079008102], "G": [48.520992279052734, 0.05255260691046715, 0.0029351452831178904], "R": [47.87550735473633, 0.19775021076202393, 0.0023990024346858263], "type": "deg2"} \ No newline at end of file +{"b": [2.6464749792362516], "g": [3.2232453214278767], "r": [2.9613987706773113]} \ No newline at end of file diff --git a/linearize-ricoh-z10-wb-5000.json b/linearize-ricoh-z10-wb-5000.json deleted file mode 100644 index 680d3f147..000000000 --- a/linearize-ricoh-z10-wb-5000.json +++ /dev/null @@ -1 +0,0 @@ -{"B": [46.82997512817383, 0.11160952597856522, 0.002294188365340233], "G": [41.67820739746094, 0.09581712633371353, 0.0028952741995453835], "R": [37.4744873046875, 0.22809027135372162, 0.002323441905900836], "type": "deg2"} \ No newline at end of file diff --git a/video_rendering_code/hdr_blender.py b/video_rendering_code/hdr_blender.py new file mode 100644 index 000000000..d6e0da3a7 --- /dev/null +++ b/video_rendering_code/hdr_blender.py @@ -0,0 +1,58 @@ +""" +Script to render IBL renders +Usage: blender -P hdr_blender.py -b +""" +import os +import random +from math import radians +from os import listdir +from os.path import isfile, join + +import bpy +import cycles +from mathutils import Euler, Matrix, Vector + + +import sys +argv = sys.argv +argv = argv[argv.index("--") + 1:] # get all args after "--" + +inp_dirs = [argv[0]] +out_dirs = [argv[1]] + +pairs = list(zip(inp_dirs,out_dirs)) +scn = bpy.context.scene +scn.render.engine = 'CYCLES' +scn.render.image_settings.file_format = 'OPEN_EXR' +scn.world.use_nodes = True +# Enabf_le GPU rendering. +# bpy.context.preferences.addons['cycles'].preferences.compute_device_type = 'CUDA' +bpy.context.preferences.addons['cycles'].preferences.get_devices() +for device in bpy.context.preferences.addons['cycles'].preferences.devices: + if "TITAN RX" == device.name: + device.use = True + else: + device.use = False +# select world node tree +wd = scn.world +nt = bpy.data.worlds[wd.name].node_tree + +for pair in pairs: + inp, out = pair + inp = inp + os.makedirs(out,exist_ok=True) + S = bpy.context.scene + envFiles = [f for f in listdir(inp) if isfile(join(inp, f)) and f.endswith('.exr')] + i = 0 + for env_map in envFiles: + env_file = join(inp, env_map) + bpy.ops.image.open(filepath=env_file, directory=inp, files=[ + {"name": env_map, "name": env_map}], show_multiview=False) + output_name = env_map.split('.')[0] +'.exr' + + backNode = nt.nodes['Environment Texture'] + backNode.image = bpy.data.images.load(env_file) + + bpy.context.scene.render.filepath = os.path.join(out, output_name) + print("Output",os.path.join(out, output_name)) + bpy.ops.render.render(write_still=True) diff --git a/video_rendering_code/render_video.sh b/video_rendering_code/render_video.sh new file mode 100644 index 000000000..507a8ebc9 --- /dev/null +++ b/video_rendering_code/render_video.sh @@ -0,0 +1,160 @@ + +export scene_name=my_apartment_ns +export output_dir=renders/${scene_name}/lantern-nerfacto/video_images +export root_dir=data/${scene_name} +export model_path=outputs/${scene_name}/lantern-nerfacto/2024-05-16_212421/config.yml + +# camera paths +# video ground: data/Henry_Gagnon_blue_et_rouge_ns/camera_paths/2024-05-16_015020.json +# still bench: 2024-05-16_015020_render +# pespective cam: 2024-05-16_015020_render3 + +# meeting room +# export camera_path_name=2024-05-16_201826_persp +# export camera_path_name=2024-05-16_201826_envmap + +# my_apartment +# export camera_path_name=2024-05-16_212421_persp +# export camera_path_name=2024-05-16_212421_env1 +# export camera_path_name=2024-05-16_212421_env2 +# export camera_path_name=2024-05-16_212421_env3 +# export camera_path_name=2024-05-16_212421_env4 +export camera_path_name=2024-05-16_212421_env5 +export camera_path=data/${scene_name}/camera_paths/${camera_path_name}.json + +ns-render camera-path --load-config $model_path --camera-path-filename $camera_path --output-path $output_dir/${camera_path_name}/well-expo/ --output-format images +ns-render camera-path --load-config $model_path --camera-path-filename $camera_path --output-path $output_dir/${camera_path_name}/fast-expo/ --output-format images --rendered_output_names rgb_fast + +echo "Combine two exposures" +python video_rendering_code/combine.py --well_dir $output_dir/${camera_path_name}/well-expo/ --fast_dir $output_dir/${camera_path_name}/fast-expo/ --out_dir $output_dir/${camera_path_name}/combined/ --do_linearization --experiment_location $root_dir + +# blender --background video_rendering_code/scene_new.blend -P video_rendering_code/hdr_blender.py -- $output_dir/${camera_path_name}/combined/ $output_dir/${camera_path_name}/render/ + + + + +######### + +export camera_path_name=2024-05-16_212421_env1 +export camera_path=data/${scene_name}/camera_paths/${camera_path_name}.json + +echo "Combine two exposures" +python video_rendering_code/combine.py --well_dir $output_dir/${camera_path_name}/well-expo/ --fast_dir $output_dir/${camera_path_name}/fast-expo/ --out_dir $output_dir/${camera_path_name}/combined/ --do_linearization --experiment_location $root_dir + + +export camera_path_name=2024-05-16_212421_env2 +export camera_path=data/${scene_name}/camera_paths/${camera_path_name}.json + +echo "Combine two exposures" +python video_rendering_code/combine.py --well_dir $output_dir/${camera_path_name}/well-expo/ --fast_dir $output_dir/${camera_path_name}/fast-expo/ --out_dir $output_dir/${camera_path_name}/combined/ --do_linearization --experiment_location $root_dir + + + + +export camera_path_name=2024-05-16_212421_env3 +export camera_path=data/${scene_name}/camera_paths/${camera_path_name}.json + +echo "Combine two exposures" +python video_rendering_code/combine.py --well_dir $output_dir/${camera_path_name}/well-expo/ --fast_dir $output_dir/${camera_path_name}/fast-expo/ --out_dir $output_dir/${camera_path_name}/combined/ --do_linearization --experiment_location $root_dir + + + + +export camera_path_name=2024-05-16_212421_env4 +export camera_path=data/${scene_name}/camera_paths/${camera_path_name}.json + +echo "Combine two exposures" +python video_rendering_code/combine.py --well_dir $output_dir/${camera_path_name}/well-expo/ --fast_dir $output_dir/${camera_path_name}/fast-expo/ --out_dir $output_dir/${camera_path_name}/combined/ --do_linearization --experiment_location $root_dir +export camera_path_name=2024-05-16_212421_env5 +export camera_path=data/${scene_name}/camera_paths/${camera_path_name}.json + +echo "Combine two exposures" +python video_rendering_code/combine.py --well_dir $output_dir/${camera_path_name}/well-expo/ --fast_dir $output_dir/${camera_path_name}/fast-expo/ --out_dir $output_dir/${camera_path_name}/combined/ --do_linearization --experiment_location $root_dir + + +#### + +# meeting room teaser + + +export scene_name=meeting_room_ns +export output_dir=renders/${scene_name}/lantern-nerfacto/video_images +export root_dir=data/${scene_name} +export model_path=outputs/${scene_name}/lantern-nerfacto/2024-05-16_201826/config.yml + +# meeting room +# export camera_path_name=2024-05-16_201826_persp +# export camera_path_name=2024-05-16_201826_persp2 + +#export camera_path_name=2024-05-16_201826_env1 +# export camera_path_name=2024-05-16_201826_env2 +# export camera_path_name=2024-05-16_201826_env3 +# export camera_path_name=2024-05-16_201826_env4 +# export camera_path_name=2024-05-16_201826_env5 +# export camera_path_name=2024-05-16_201826_env6 +# export camera_path_name=2024-05-16_201826_env7 +export camera_path_name=2024-05-16_201826_env8 + +export camera_path=data/${scene_name}/camera_paths/${camera_path_name}.json + +ns-render camera-path --load-config $model_path --camera-path-filename $camera_path --output-path $output_dir/${camera_path_name}/well-expo/ --output-format images +ns-render camera-path --load-config $model_path --camera-path-filename $camera_path --output-path $output_dir/${camera_path_name}/fast-expo/ --output-format images --rendered_output_names rgb_fast + +echo "Combine two exposures" +python lantern_scripts/combine.py --well_dir $output_dir/${camera_path_name}/well-expo/ --fast_dir $output_dir/${camera_path_name}/fast-expo/ --out_dir $output_dir/${camera_path_name}/combined/ --do_linearization --experiment_location $root_dir + +# Fig. 5 - ABO objects + +export scene_name=LVSN_LAB_OpenSFM +export output_dir=renders/${scene_name}/lantern-nerfacto/video_images +export root_dir=data/${scene_name} +export model_path=outputs/${scene_name}/lantern-nerfacto/step_2/config.yml + +# export camera_path_name=persp1 +export camera_path_name=persp2 +# export camera_path_name=env1 +# export camera_path_name=env2 +# export camera_path_name=env3 + +export camera_path=data/${scene_name}/camera_paths/${camera_path_name}.json + +ns-render camera-path --load-config $model_path --camera-path-filename $camera_path --output-path $output_dir/${camera_path_name}/well-expo/ --output-format images +ns-render camera-path --load-config $model_path --camera-path-filename $camera_path --output-path $output_dir/${camera_path_name}/fast-expo/ --output-format images --rendered_output_names rgb_fast + +echo "Combine two exposures" +python lantern_scripts/combine.py --well_dir $output_dir/${camera_path_name}/well-expo/ --fast_dir $output_dir/${camera_path_name}/fast-expo/ --out_dir $output_dir/${camera_path_name}/combined/ --do_linearization --experiment_location $root_dir + +######### Render videos ############ + +# LVSN lab upstairs + + +export scene_name=LVSN_LAB_OpenSFM +export model_type=hdr-nerfacto +# export model_path=outputs/${scene_name}/lantern-nerfacto/step_2/config.yml +export output_dir=renders/${scene_name}/${model_type}/video_images +export model_path=outputs/${scene_name}/${model_type}/2024-05-18_143132/config.yml +export camera_path_name=video3 +export camera_path=data/${scene_name}/camera_paths/${camera_path_name}.json + +ns-render camera-path --load-config $model_path --camera-path-filename $camera_path --output-path $output_dir/${camera_path_name}/well-expo/ --output-format images +ns-render camera-path --load-config $model_path --camera-path-filename $camera_path --output-path $output_dir/${camera_path_name}/fast-expo/ --output-format images --rendered_output_names rgb_fast + +echo "Combine two exposures" +python lantern_scripts/combine.py --well_dir $output_dir/${camera_path_name}/well-expo/ --fast_dir $output_dir/${camera_path_name}/fast-expo/ --out_dir $output_dir/${camera_path_name}/combined/ --do_linearization --experiment_location $root_dir + +# for Yannick +zip -rq combined_${scene_name}_${model_type}.zip $output_dir/${camera_path_name}/combined + +blender --background video_rendering_code/scene_new.blend -P video_rendering_code/hdr_blender.py -- $output_dir/${camera_path_name}/combined/ $output_dir/${camera_path_name}/render/ + +mkdir $output_dir/${camera_path_name}/render_tonemapped +python lantern_scripts/tonemap.py --data_dir $output_dir/${camera_path_name}/render/ --out_dir $output_dir/${camera_path_name}/render_tonemapped +python lantern_scripts/mp4_maker.py --input_Addr $output_dir/${camera_path_name}/render_tonemapped --out_file $output_dir/${camera_path_name}/video_render.mp4 + +## + +export scene_name=meeting_room_ns + +export model_path=outputs/${scene_name}/lantern-nerfacto/2024-05-16_201826/config.yml +