Two exposures - with weighted loss

This commit is contained in:
mokad6 committed 2023-11-01 16:12:36 -04:00
1 parent 1cea0fe30e
commit 836ac8b223
12 files changed
+544 -18

No files matched your search

+32
View File
@@ -0,0 +1,32 @@
import os
import numpy as np
from tqdm import tqdm
from envmap import EnvironmentMap, rotation_matrix
from hdrio import imwrite
# Outdoor Table
LDR_dir = '/mnt/data/arch_level1/panoramas_first_100/'
HDR_dir = '/mnt/data/arch_level1/panoramas_256x512_hdr/'
output_dir = '/mnt/data/arch_level1/panoramas_LDR_HDR_small/'
imgFiles = [os.path.join(LDR_dir, f) for f in os.listdir(LDR_dir) if os.path.isfile(os.path.join(LDR_dir, f)) and f.endswith('.png')]
imgFiles.sort()
for ldr_addr in tqdm(imgFiles):
ldr_pano = EnvironmentMap(ldr_addr, 'latlong')
hdr_addr = ldr_addr.replace('.png', '.exr')
hdr_addr = hdr_addr.replace(LDR_dir, HDR_dir)
assert os.path.exists(hdr_addr), "HDR file does not exists!!!"
hdr_pano = EnvironmentMap(hdr_addr, 'latlong')
hdr_pano.resize(1920)
out_pano = ldr_pano.data ** 2.2
list_hdr = hdr_pano.data > 1.0
print("list_hdr.shape: ", list_hdr.shape, "list_hdr.shape: ", ldr_pano.data.shape)
out_pano[list_hdr] = hdr_pano.data[list_hdr]
out_addr = hdr_addr.replace(HDR_dir, output_dir)
imwrite(out_pano, out_addr)
print("Done!")
+166
View File
@@ -0,0 +1,166 @@
import argparse
import numpy as np
import os
from tqdm import tqdm
from pathlib import Path
from PIL import Image
import copy
import cv2
import json
from pathlib import Path
os.environ["OPENCV_IO_ENABLE_OPENEXR"] = "1"
def weighting_function(pixels, zmin, zmax):
threshold = 0.5 * (zmin + zmax)
weights = np.zeros(pixels.shape) + np.finfo(np.float64).eps
# mask_lowerbound = zmin <= pixels <= threshold
mask_lowerbound = np.greater_equal(pixels, zmin) & np.less_equal(pixels, threshold)
weights[mask_lowerbound] = pixels[mask_lowerbound] - zmin
# mask_upperbound = threshold <= pixels <= zmax
mask_upperbound = np.greater_equal(pixels, threshold) & np.less_equal(pixels, zmax)
weights[mask_upperbound] = zmax - pixels[mask_upperbound]
# sums_R = np.sum(weights[:,:,0])
# sums_G = np.sum(weights[:,:,1])
# sums_B = np.sum(weights[:,:,2])
return weights #/ [sums_R, sums_G, sums_B]
def apply_the_exposure_get_weights(hdr_data, exposure):
e1_data = np.clip(hdr_data * exposure, 0, 1)
e1_data = (e1_data * 255.0).astype("uint8")
e1_data = e1_data.astype('float32')
e1_data = e1_data / 255.0
weights = weighting_function(e1_data, 0.05, 0.95)
e1_data_HDR = e1_data / exposure
return e1_data, e1_data_HDR, weights
def make_weights_binary(pixels, weights, thresh_min, thresh_max, threshold_weights = False):
outputs = np.zeros(pixels.shape)
if threshold_weights:
weights_e1_if_case = copy.deepcopy(weights)
else:
weights_e1_if_case = copy.deepcopy(pixels)
# masked = thresh_min <= weights_e1_if_case <= thresh_max
masked = np.greater_equal(weights_e1_if_case, thresh_min) & np.less_equal(weights_e1_if_case, thresh_max)
outputs[masked] = 1.0
outputs_unit = (255. * outputs).astype('uint8')
return outputs_unit
def load_from_json(filename: Path):
"""Load a dictionary from a JSON filename.
Args:
filename: The filename to load from.
"""
assert filename.suffix == ".json"
with open(filename, encoding="UTF-8") as file:
return json.load(file)
def write_to_json(filename: Path, content: dict):
"""Write data to a JSON file.
Args:
filename: The filename to write to.
content: The dictionary data to write.
"""
assert filename.suffix == ".json"
# with open(filename, "w", encoding="UTF-8") as file:
# json.dump(content, file)
with open(filename, "w", encoding="utf-8") as f:
json.dump(content, f, indent=4)
def process_one_image(pixels, out_dir, name, has_exposure, exposure_value, zmin, zmax):
if args.has_exposure:
e1_ldr, e1_hdr, weights_e1 = apply_the_exposure_get_weights(pixels, exposure_value)
weights_e1_unit = make_weights_binary(e1_ldr, weights_e1, zmin, zmax)
mask_e1 = weights_e1_unit[:, :, 0] & weights_e1_unit[:, :, 1] & weights_e1_unit[:, :, 2]
e1_mask_addr = os.path.join(out_dir, name + '.png')
# e1_mask_addr = os.path.join(args.out_dir, pano_name + '_e1.png')
Image.fromarray(mask_e1).save(e1_mask_addr)
else:
e1_hdr = pixels
# out_addr = os.path.join(args.out_dir, pano_name + '_e1.exr')
out_addr = os.path.join(out_dir, name + '.exr')
cv2.imwrite(out_addr, e1_hdr)
def process_all_images(address, out_dir, prefix, has_exposure, exposure_value, zmin, zmax):
images = [os.path.join(address, f) for f in os.listdir(address) if os.path.isfile(os.path.join(address, f)) and f.endswith('.exr')]
images = sorted(images)
for pano_addr in tqdm(images):
# original_pano = EnvironmentMap(pano_addr, 'latlong')
# pixels = original_pano.data[:, :, 0:3]
original_pano = np.array(cv2.imread(str(pano_addr), cv2.IMREAD_UNCHANGED)).astype("float32")
pixels = original_pano[:, :, 0:3]
pano_file = os.path.basename(pano_addr)
pano_name = pano_file[:-4]
name = prefix + pano_name
process_one_image(pixels, out_dir, name, has_exposure, exposure_value, zmin, zmax)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--left_dir', type=str, default='/mnt/workspace/share_w_Junming/Dual_Cameras/left/')
parser.add_argument('--right_dir', type=str, default='/mnt/workspace/share_w_Junming/Dual_Cameras/right/')
parser.add_argument('--out_dir', type=str, default='/mnt/workspace/share_w_Junming/Dual_Cameras/lantern-HDR/')
parser.add_argument('--has_exposure', action='store_true')
parser.add_argument('--exposure1', type=float, default=1.0)
parser.add_argument('--exposure2', type=float, default=0.009)
args = parser.parse_args()
assert os.path.isfile(args.left_dir + '/transforms.json'), 'The JSON files for left set is missing!'
assert os.path.isfile(args.right_dir + '/transforms.json'), 'The JSON files for right set is missing!'
assert os.path.isdir(args.left_dir + '/HDR_Normal/'), 'The data folder for left set is missing!'
assert os.path.isdir(args.right_dir + '/HDR_Normal/'), 'The data folder for right set is missing!'
assert not os.path.isdir(args.out_dir), 'The output folder exists!! Take care of it first!'
os.mkdir(args.out_dir)
os.mkdir(args.out_dir + 'HDR_Normal/')
cam_params_lhs = load_from_json( Path(args.left_dir + '/transforms.json') )
cam_params_combine = {}
cam_params_combine["camera_angle_x"] = cam_params_lhs["camera_angle_x"]
cam_params_combine["frames"] = []
for frame in tqdm(cam_params_lhs["frames"]):
file_path = 'lhs_' + frame["file_path"].split('/')[-1]
transform_matrix = frame["transform_matrix"]
cam_params_combine["frames"].append(
{
"file_path": file_path,
"transform_matrix": transform_matrix
}
)
# TODO adding exposures
cam_params_rhs = load_from_json( Path(args.right_dir + '/transforms.json') )
for frame in tqdm(cam_params_rhs["frames"]):
file_path = 'rhs_' + frame["file_path"].split('/')[-1]
transform_matrix = frame["transform_matrix"]
cam_params_combine["frames"].append(
{
"file_path": file_path,
"transform_matrix": transform_matrix
}
)
# TODO adding exposures
write_to_json(Path(args.out_dir + '/transforms_all.json'), cam_params_combine)
process_all_images(args.left_dir + '/HDR_Normal/', args.out_dir + '/HDR_Normal/', 'lhs_', args.has_exposure, args.exposure1, 0.0, 0.95)
process_all_images(args.right_dir + '/HDR_Normal/', args.out_dir + '/HDR_Normal/', 'rhs_', args.has_exposure, args.exposure2, 0.2, 1.0)
print("Done!")
+102
View File
@@ -0,0 +1,102 @@
import argparse
from envmap import EnvironmentMap
import numpy as np
import os
from tqdm import tqdm
from pathlib import Path
from hdrio import imwrite
from PIL import Image
import copy
def weighting_function(pixels, zmin, zmax):
threshold = 0.5 * (zmin + zmax)
weights = np.zeros(pixels.shape) + np.finfo(np.float64).eps
# mask_lowerbound = zmin <= pixels <= threshold
mask_lowerbound = np.greater_equal(pixels, zmin) & np.less_equal(pixels, threshold)
weights[mask_lowerbound] = pixels[mask_lowerbound] - zmin
# mask_upperbound = threshold <= pixels <= zmax
mask_upperbound = np.greater_equal(pixels, threshold) & np.less_equal(pixels, zmax)
weights[mask_upperbound] = zmax - pixels[mask_upperbound]
# sums_R = np.sum(weights[:,:,0])
# sums_G = np.sum(weights[:,:,1])
# sums_B = np.sum(weights[:,:,2])
return weights #/ [sums_R, sums_G, sums_B]
def apply_the_exposure_get_weights(hdr_data, exposure):
e1_data = np.clip(hdr_data * exposure, 0, 1)
e1_data = (e1_data * 255.0).astype("uint8")
e1_data = e1_data.astype('float32')
e1_data = e1_data / 255.0
weights = weighting_function(e1_data, 0.05, 0.95)
e1_data_HDR = e1_data / exposure
return e1_data, e1_data_HDR, weights
def make_weights_binary(pixels, weights, thresh_min, thresh_max, threshold_weights = False):
outputs = np.zeros(pixels.shape)
if threshold_weights:
weights_e1_if_case = copy.deepcopy(weights)
else:
weights_e1_if_case = copy.deepcopy(pixels)
# masked = thresh_min <= weights_e1_if_case <= thresh_max
masked = np.greater_equal(weights_e1_if_case, thresh_min) & np.less_equal(weights_e1_if_case, thresh_max)
outputs[masked] = 1.0
outputs_unit = (255. * outputs).astype('uint8')
return outputs_unit
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--data_dir', type=str, default='/mnt/data/nerfstudio_ds/synthetic_classroom/cameras_w_baseline/left/')
parser.add_argument('--out_dir', type=str, default='/mnt/data/nerfstudio_ds/synthetic_classroom/cameras_w_baseline/all/')
args = parser.parse_args()
e1 = 1.0 # None # Left
e2 = None # 0.009 # Right
test_images = []
for path in Path(args.data_dir).rglob('*.exr'):
test_images.append(path)
for pano_addr in tqdm(test_images):
original_pano = EnvironmentMap(pano_addr, 'latlong')
pixels = original_pano.data[:, :, 0:3]
pano_file = os.path.basename(pano_addr)
pano_name = pano_file[:-4]
if e1 is not None:
e1_ldr, e1_hdr, weights_e1 = apply_the_exposure_get_weights(pixels, e1)
weights_e1_unit = make_weights_binary(e1_ldr, weights_e1, 0.0, 0.95)
mask_e1 = weights_e1_unit[:, :, 0] & weights_e1_unit[:, :, 1] & weights_e1_unit[:, :, 2]
# out_addr = os.path.join(args.out_dir, pano_name + '_e1.exr')
out_addr = os.path.join(args.out_dir, 'lhs_' + pano_name + '.exr')
imwrite(e1_hdr, out_addr)
e1_mask_addr = os.path.join(args.out_dir, 'lhs_' + pano_name + '.png')
# e1_mask_addr = os.path.join(args.out_dir, pano_name + '_e1.png')
Image.fromarray(mask_e1).save(e1_mask_addr)
if e2 is not None:
e2_ldr, e2_hdr, weights_e2 = apply_the_exposure_get_weights(pixels, e2)
weights_e2_unit = make_weights_binary(e2_ldr, weights_e2, 0.2, 1.0)
mask_e2 = weights_e2_unit[:, :, 0] & weights_e2_unit[:, :, 1] & weights_e2_unit[:, :, 2]
# out_addr = os.path.join(args.out_dir, pano_name + '_e2.exr')
out_addr = os.path.join(args.out_dir, 'rhs_' + pano_name + '.exr')
imwrite(e2_hdr, out_addr)
# e2_mask_addr = os.path.join(args.out_dir, pano_name + '_e2.png')
e2_mask_addr = os.path.join(args.out_dir, 'rhs_' + pano_name + '.png')
Image.fromarray(mask_e2).save(e2_mask_addr)
# e1_e2_combined = (weights_e2 * e2_data + weights_e1 * e1_data) / (weights_e1 + weights_e2)
# out_addr = os.path.join(args.out_dir, pano_name + '_combine.exr')
# imwrite(e1_e2_combined, out_addr)
print("Done!")
+79
View File
@@ -0,0 +1,79 @@
"""
Script to render IBL renders
Usage: blender <blend file> -P hdr_blender.py -b
"""
import os
import random
from math import radians
from os import listdir
from os.path import isfile, join
import bpy
import cycles
from mathutils import Euler, Matrix, Vector
# path = 'pano-videos/results/panohdr_log/chess_room/'
path = './'
datasets = ['chess_room','old_class_spot','old_class_neon','small_class'\
,'stairway','cafeteria']
# # out_dirs = ['prof_kitchen/','prof_living_room/','grandma_attic/'\
# # ,'grandma_guest_room/','chess_room/'\
# # ,'old_class_spot/','old_class_neon/','small_class/'\
# # ,'music_room/']
# # [f'gt/{data}/' for data in datasets] + [f'ldr/{data}/' for data in datasets]
# # [f'blender_multi/gt/{data}/' for data in datasets] + [f'blender_multi/ldr/{data}/' for data in datasets] \
inp_dirs = ['/mnt/workspace/lantern/renders/lantern-dual-exposure_no_mask/2023-11-01_022121/']
# [f'results/ft_momo/{data}/mask_pred/' for data in datasets] + \
# [f'results/1hdr/{data}/mask_pred/' for data in datasets]
out_dirs = ['/mnt/workspace/lantern/renders/lantern-dual-exposure_no_mask/2023-11-01_022121_w_objects/']
# [f'blender_multi/ft_momo/{data}/mask_pred/' for data in datasets] + \
# [f'blender_multi/1hdr/{data}/mask_pred/' for data in datasets]
# inp_dirs = ['final_0/exr/','final_1/exr/','final_2/exr/']
# out_dirs = ['final_0_render/','final_1_render/','final_2_render/']
# out_dirs = ['old_class_spot/render_ldr_500000/render/','chess_room/render_ldr_500000/render/',
# 'cafeteria/render_ldr_500000/render/','stairway/render_ldr_500000/render/',
# 'small_class/render_ldr_500000/render/','cafeteria/render_ldr_500000/render/',\
# 'prof_living_room/render_ldr_500000/render/']
pairs = list(zip(inp_dirs,out_dirs))
scn = bpy.context.scene
scn.render.engine = 'CYCLES'
scn.render.image_settings.file_format = 'OPEN_EXR'
scn.world.use_nodes = True
# Enabf_le GPU rendering.
bpy.context.preferences.addons['cycles'].preferences.compute_device_type = 'CUDA'
bpy.context.preferences.addons['cycles'].preferences.get_devices()
for device in bpy.context.preferences.addons['cycles'].preferences.devices:
if "TITAN V" == device.name:
device.use = True
else:
device.use = False
# select world node tree
wd = scn.world
nt = bpy.data.worlds[wd.name].node_tree
for pair in pairs:
inp,out = pair
inp = inp
out = os.path.join(path,out)
os.makedirs(out,exist_ok=True)
S = bpy.context.scene
envFiles = [f for f in listdir(inp) if isfile(join(inp, f)) and f.endswith('.exr')]
print(envFiles)
i = 0
for env_map in envFiles:
env_file = join(inp, env_map)
bpy.ops.image.open(filepath=env_file, directory=inp, files=[
{"name": env_map, "name": env_map}], show_multiview=False)
output_name = "render_" + env_map.split('.')[0] +'.exr'
# print(output_name)
# bpy.ops.image.save_as(save_as_render=True, copy=True, filepath=join(out_addr, output_name), show_multiview=False, use_multiview=False)
backNode = nt.nodes['Environment Texture']
backNode.image = bpy.data.images.load(env_file)
bpy.context.scene.render.filepath = os.path.join(out, output_name)
print("Output",os.path.join(out, output_name))
bpy.ops.render.render(write_still=True)
+42
View File
@@ -0,0 +1,42 @@
import argparse
from envmap import EnvironmentMap, rotation_matrix
import numpy as np
import os
from tqdm import tqdm
from PIL import Image
from pathlib import Path
import imageio
def tonemap(img, gamma=2.4):
"""Apply gamma, then clip between 0 and 1, finally convert to uint8 [0,255]"""
return (np.clip(np.power(img,1/gamma), 0.0, 1.0)*255).astype('uint8')
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--data_dir', type=str, default='/Volumes/Momo/Adobe_2023/Evaluation/test_sets/weber_test_set_original_hdr_panos/')
parser.add_argument('--out_dir', type=str, default='/Users/momo/Desktop/camera_ready/FID_test_set_256x512/')
parser.add_argument("--resolution", type=int, default=256)
args = parser.parse_args()
test_images = []
for path in Path(args.data_dir).rglob('*.exr'):
test_images.append(path)
imageio.plugins.freeimage.download()
for pano_addr in tqdm(test_images):
# original_pano = imageio.imread(pano_addr, format='HDR-FI') # EnvironmentMap(pano_addr, 'latlong')
# # gt_ldr = tonemap(reexpose_hdr(original_pano.data)[0])
# gt_ldr = tonemap(reexpose_hdr(original_pano)[0])
original_pano = EnvironmentMap(pano_addr, 'latlong')
# gt_ldr = tonemap(reexpose_hdr(original_pano.data)[0], gamma=2.2) # for computing the FID in table 1, sue tune mapping of 2.2
# gt_ldr = tonemap(reexpose_hdr(original_pano)[0])
gt_ldr = tonemap(original_pano.data, gamma=2.2) # No Need to re-expose the StyleLight and EverLight
pano_file = os.path.basename(pano_addr)
pano_file = pano_file.replace('.exr', '.png')
gt_ldr_addr = os.path.join(args.out_dir, pano_file)
# Image.fromarray(gt_ldr).resize((2 * args.resolution, args.resolution), resample=Image.LANCZOS).save(gt_ldr_addr)
Image.fromarray(gt_ldr).save(gt_ldr_addr)
print("Done!")