diff --git a/lantern_scripts/calculate_metrics.py b/lantern_scripts/calculate_metrics.py index c4e372962..24b3d6794 100644 --- a/lantern_scripts/calculate_metrics.py +++ b/lantern_scripts/calculate_metrics.py @@ -2,6 +2,8 @@ import subprocess import argparse import os import shutil +import sys + if __name__ == '__main__': argparser = argparse.ArgumentParser() @@ -20,16 +22,35 @@ if __name__ == '__main__': metrics_dir = os.path.join(args.input_dir, "metrics") renders_dir = os.path.join(metrics_dir, "images") + renders_well = os.path.join(renders_dir, args.metric_name) + if os.path.exists(renders_well): + shutil.rmtree(renders_well) + renders_fast = os.path.join(renders_dir, args.metric_name + "_fast") + if os.path.exists(renders_fast): + shutil.rmtree(renders_fast) + renders_lin = os.path.join(renders_dir, args.metric_name + "_lin") + if os.path.exists(renders_lin): + shutil.rmtree(renders_lin) + renders_lin_HDR = os.path.join(renders_dir, args.metric_name + "_lin_HDR") + if os.path.exists(renders_lin_HDR): + shutil.rmtree(renders_lin_HDR) + lin_renders = os.path.join(renders_dir, args.metric_name + "_lin_renders") + if os.path.exists(lin_renders): + shutil.rmtree(lin_renders) gt_dir = os.path.join(args.input_dir, "GT") gt_hdr = os.path.join(gt_dir, "GT_hdr") + if os.path.exists(gt_hdr): + shutil.rmtree(gt_hdr) gt_exr = os.path.join(gt_dir, "GT_exr") gt_exr_renders = os.path.join(gt_dir, "GT_exr_renders") + if os.path.exists(gt_exr_renders): + shutil.rmtree(gt_exr_renders) # 1. Get ground truth transformations: print("Running ns-process-data to get GT transformations...") diff --git a/lantern_scripts/calculate_metrics_for_step_1.py b/lantern_scripts/calculate_metrics_for_step_1.py index e78dc4f02..94f3c4b9c 100644 --- a/lantern_scripts/calculate_metrics_for_step_1.py +++ b/lantern_scripts/calculate_metrics_for_step_1.py @@ -2,6 +2,8 @@ import subprocess import argparse import os import shutil +import sys + if __name__ == '__main__': argparser = argparse.ArgumentParser() @@ -20,16 +22,35 @@ if __name__ == '__main__': metrics_dir = os.path.join(args.input_dir, "metrics") renders_dir = os.path.join(metrics_dir, "images") + renders_well = os.path.join(renders_dir, args.metric_name) + if os.path.exists(renders_well): + shutil.rmtree(renders_well) + renders_fast = os.path.join(renders_dir, args.metric_name + "_fast") + if os.path.exists(renders_fast): + shutil.rmtree(renders_fast) + renders_lin = os.path.join(renders_dir, args.metric_name + "_lin") + if os.path.exists(renders_lin): + shutil.rmtree(renders_lin) + renders_lin_HDR = os.path.join(renders_dir, args.metric_name + "_lin_HDR") + if os.path.exists(renders_lin_HDR): + shutil.rmtree(renders_lin_HDR) + lin_renders = os.path.join(renders_dir, args.metric_name + "_lin_renders") + if os.path.exists(lin_renders): + shutil.rmtree(lin_renders) gt_dir = os.path.join(args.input_dir, "GT") gt_hdr = os.path.join(gt_dir, "GT_hdr") + if os.path.exists(gt_hdr): + shutil.rmtree(gt_hdr) gt_exr = os.path.join(gt_dir, "GT_exr") gt_exr_renders = os.path.join(gt_dir, "GT_exr_renders") + if os.path.exists(gt_exr_renders): + shutil.rmtree(gt_exr_renders) # 1. Get ground truth transformations: print("Running ns-process-data to get GT transformations...") @@ -156,3 +177,38 @@ if __name__ == '__main__': "--hdr_dir", gt_exr, "--output_dir", gt_hdr ], check=True) + + # 9. Convert EXR panos to LDR format: + print("Converting EXR panos to LDR visualization...") + subprocess.run([ + "python", "lantern_scripts/tonemap.py", + "--data_dir", renders_lin, + "--out_dir", renders_lin + ], check=True) + subprocess.run([ + "python", "lantern_scripts/tonemap.py", + "--data_dir", gt_exr, + "--out_dir", gt_exr + ], check=True) + subprocess.run([ + "python", "lantern_scripts/tonemap.py", + "--data_dir", lin_renders, + "--out_dir", lin_renders + ], check=True) + subprocess.run([ + "python", "lantern_scripts/tonemap.py", + "--data_dir", gt_exr_renders, + "--out_dir", gt_exr_renders + ], check=True) + + # 9. Convert EXR panos to LDR format: + print("Preparing the HTML visualization...") + subprocess.run([ + "python", "lantern_scripts/results_visualization.py", + "--gt_dir", gt_exr, + "--gt_render_dir", gt_exr_renders, + "--pandora_dir", renders_lin, + "--pandora_render_dir", lin_renders, + "--out_dir", args.input_dir, + "--name", args.metric_name + ], check=True) \ No newline at end of file diff --git a/lantern_scripts/create_GT_folder.py b/lantern_scripts/create_GT_folder.py index d961c1397..e9fc3231e 100644 --- a/lantern_scripts/create_GT_folder.py +++ b/lantern_scripts/create_GT_folder.py @@ -2,12 +2,16 @@ import os import shutil import re import argparse +import subprocess +from pathlib import Path + def copy_files(input_dir): brackets = os.listdir(input_dir) target_exr = os.path.join(input_dir, "GT_exr") target_jpg = os.path.join(input_dir, "GT_jpg") + target_jpg_resize = os.path.join(input_dir, "GT_jpg_resize") os.makedirs(target_exr, exist_ok=True) os.makedirs(target_jpg, exist_ok=True) @@ -36,6 +40,24 @@ def copy_files(input_dir): shutil.copy2(source_file, destination_file) print(f"Copied JPG: {source_file} -> {destination_file}") + # Run a simple command + print("Copying target_jpg to target_jpg_resize...") + result = subprocess.run(["cp", "-r", target_jpg, target_jpg_resize], capture_output=True, text=True) + print(result) + + print("Resizing target_jpg_resize...") + result = subprocess.run(["find", target_jpg_resize, "-iname", "*.jpg", "-exec", "convert", "{}", "-verbose", "-resize", "3840x1920>", "{}", ";"], capture_output=True, text=True) + print(result) + + gt_path = Path(input_dir) + sfm_path = os.path.join(gt_path.parent, 'sfm/images') + for file in os.listdir(target_jpg_resize): + source_file = os.path.join(target_jpg_resize, file) + destination_file = os.path.join(sfm_path, file) + shutil.copy2(source_file, destination_file) + print(f"Copied JPG: {source_file} -> {destination_file}") + + if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--input_dir', type=str, required=True, @@ -43,3 +65,6 @@ if __name__ == '__main__': args = parser.parse_args() copy_files(args.input_dir) + + + diff --git a/lantern_scripts/data_visualization.py b/lantern_scripts/data_visualization.py index 50c022dfe..3ee7114f8 100644 --- a/lantern_scripts/data_visualization.py +++ b/lantern_scripts/data_visualization.py @@ -2,6 +2,7 @@ from os import makedirs from html4vision import Col, imagetable import argparse import json +import os if __name__ == "__main__": @@ -9,13 +10,80 @@ if __name__ == "__main__": parser.add_argument('--input_dir', type=str, default='/mnt/data/nerfstudio_ds/real_data/coffeeroom_1102H/GT/GT_exr') args = parser.parse_args() + left_e1_dir = args.input_dir + 'left_e1/' + left_e1_mask_dir = args.input_dir + 'masks/left_e1/' + right_e2_dir = args.input_dir + 'right_e2/' + right_e2_mask_dir = args.input_dir + 'masks/right_e2/' + + imgs = [f for f in os.listdir(left_e1_dir) if f.endswith(".png")] + + left_e1 = [] + left_mask = [] + right_e2 = [] + right_mask = [] + for img in imgs: + left_e1.append( os.path.join(left_e1_dir, img) ) + left_mask.append( os.path.join(left_e1_mask_dir, img + '.masked.png') ) + + right_e2.append( os.path.join(right_e2_dir, img) ) + right_mask.append( os.path.join(right_e2_mask_dir, img + '.masked.png') ) + cols = [ - Col('id1', 'ID'), # 1-based indexing - Col('img', 'Left Pano', args.input_dir + 'left_e1/output_*.png'), - Col('img', 'Left Mask', args.input_dir + 'masks/left_e1/output_*.png.masked.png'), - Col('img', 'Right Pano', args.input_dir + 'right_e2/output_*.png'), - Col('img', 'Right Mask', args.input_dir + 'masks/right_e2/output_*.png.masked.png'), + Col('id1', 'ID'), + Col('text', 'Name', imgs), + Col('img', 'Left Pano', left_e1), + Col('img', 'Left Mask', left_mask), + Col('img', 'Right Pano', right_e2), + Col('img', 'Right Mask', right_mask), ] - # imagetable(cols, 'another-dir/pathrep.html', 'Path Replace Example', pathrep=('/mnt/data', '/home-local2/mokad6.extra.nobkp')) - imagetable(cols, args.input_dir + 'data_index.html', 'NeRF sequence', pathrep=(args.input_dir, './'), imscale=0.1, zebra=True, sticky_header=True, overlay_toggle=True) \ No newline at end of file + imagetable( + cols, + args.input_dir + 'data_index.html', + 'NeRF sequence', + pathrep=(args.input_dir, './'), + sort_style='materialize', + imscale=0.1, + zebra=True, + sticky_header=True, + overlay_toggle=True + ) + + left_sfm_dir = args.input_dir + 'left_sfm/' + left_sfm_mask_dir = args.input_dir + 'masks/left_sfm/' + right_sfm_dir = args.input_dir + 'right_sfm/' + right_sfm_mask_dir = args.input_dir + 'masks/right_sfm/' + + imgs = [f for f in os.listdir(left_sfm_dir) if f.endswith(".png")] + + left_sfm = [] + left_mask = [] + right_sfm = [] + right_mask = [] + for img in imgs: + left_sfm.append( os.path.join(left_sfm_dir, img) ) + left_mask.append( os.path.join(left_sfm_mask_dir, img + '.masked.png') ) + + right_sfm.append( os.path.join(right_sfm_dir, img) ) + right_mask.append( os.path.join(right_sfm_mask_dir, img + '.masked.png') ) + + cols = [ + Col('id1', 'ID'), + Col('text', 'Name', imgs), + Col('img', 'Left Pano', left_sfm), + Col('img', 'Left Mask', left_mask), + Col('img', 'Right Pano', right_sfm), + Col('img', 'Right Mask', right_mask), + ] + + imagetable( + cols, + args.input_dir + 'sfm_index.html', + 'SFM sequence', + pathrep=(args.input_dir, './'), + sort_style='materialize', + imscale=0.1, + zebra=True, + sticky_header=True, + overlay_toggle=True + ) \ No newline at end of file diff --git a/lantern_scripts/debevec_bracket_merge.py b/lantern_scripts/debevec_bracket_merge.py index c3892a397..cb659777a 100644 --- a/lantern_scripts/debevec_bracket_merge.py +++ b/lantern_scripts/debevec_bracket_merge.py @@ -64,6 +64,7 @@ if __name__ == "__main__": for data_dir in GT_directories: test_images = [] + print("data_dir: ", data_dir) for path in Path(data_dir).rglob('*.JPG'): test_images.append(path) diff --git a/lantern_scripts/process_videos_for_sfm.py b/lantern_scripts/process_videos_for_sfm.py index 8a6941249..6feaa22a9 100644 --- a/lantern_scripts/process_videos_for_sfm.py +++ b/lantern_scripts/process_videos_for_sfm.py @@ -18,3 +18,5 @@ if __name__ == '__main__': if args.gt: subprocess.run(["python", "lantern_scripts/debevec_bracket_merge.py", "--data_dir", args.input_dir + "GT/", "--experiment_location", args.input_dir + "data/"], check=True) subprocess.run(["python", "lantern_scripts/create_GT_folder.py", "--input_dir", args.input_dir + "GT/"], check=True) + + subprocess.run(["python", "lantern_scripts/data_visualization.py", "--input_dir", args.input_dir + "data/"], check=True) diff --git a/lantern_scripts/sync_and_trim.py b/lantern_scripts/sync_and_trim.py index 4f6d0b975..f3cb29d7e 100644 --- a/lantern_scripts/sync_and_trim.py +++ b/lantern_scripts/sync_and_trim.py @@ -4,6 +4,9 @@ import subprocess from pydub import AudioSegment import numpy as np from scipy.signal import find_peaks +import json +import re + def extract_audio(video_path, audio_path): command = [ @@ -15,6 +18,7 @@ def extract_audio(video_path, audio_path): ] subprocess.run(command) + def detect_hand_claps(audio_path, threshold=0.5, min_distance=10000): audio = AudioSegment.from_wav(audio_path) samples = np.array(audio.get_array_of_samples()) @@ -30,6 +34,7 @@ def detect_hand_claps(audio_path, threshold=0.5, min_distance=10000): return clap_times + def trim_video(video_path, start_time, end_time, output_path): command = [ "ffmpeg", @@ -37,10 +42,14 @@ def trim_video(video_path, start_time, end_time, output_path): "-ss", str(start_time), "-to", str(end_time), "-c", "copy", + "-map", "0", + "-map_metadata", "0", + "-movflags", "use_metadata_tags", output_path ] subprocess.run(command) + def get_video_length(video_path): command = [ "ffprobe", @@ -54,6 +63,48 @@ def get_video_length(video_path): result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) return float(result.stdout.strip()) + +def extract_imu_data(video_path): + # Run exiftool with -ee to extract per-frame metadata + cmd = ['exiftool', '-ee2', '-n', '-G1', '-a', '-u', video_path] + try: + # Execute the command and capture the output + result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) #, text=True, check=True) + output = result.stdout.decode('latin-1') + except subprocess.CalledProcessError as e: + print(f"Error executing ExifTool: {e.stderr}") + return + + accel_data = [] + gyro_data = [] + + # Split the output into lines for processing + lines = output.splitlines() + i = 0 + while i < len(lines): + line = lines[i] + if 'Accelerometer' in line: + match = re.search(r': ([\-\d\.]+) ([\-\d\.]+) ([\-\d\.]+)', line) + if match: + if i + 1 < len(lines): + if 'Time Stamp' in lines[i + 1]: + time_match = re.search(r': ([\-\d\.]+)', lines[i + 1]) + accel_data.append([float(time_match.group(1)), float(match.group(1)), float(match.group(2)), float(match.group(3))]) + i += 1 + elif 'Magnetic Field' in line: + match = re.search(r': ([\-\d\.]+) ([\-\d\.]+) ([\-\d\.]+)', line) + if match: + if i + 1 < len(lines): + if 'Time Stamp' in lines[i + 1]: + time_match = re.search(r': ([\-\d\.]+)', lines[i + 1]) + gyro_data.append([float(time_match.group(1)), float(match.group(1)), float(match.group(2)), float(match.group(3))]) + i += 1 + else: + i += 1 + + return accel_data, gyro_data + + if __name__ == '__main__': argparser = argparse.ArgumentParser() argparser.add_argument("--input_dir", type=str, default="/mnt/data/scene/", help="Directory containing all the 4 videos") @@ -90,10 +141,30 @@ if __name__ == '__main__': start_clap_1, end_clap_1 = clap_times_1[0], clap_times_1[-1] start_clap_2, end_clap_2 = clap_times_2[0], clap_times_2[-1] + accel_data, gyro_data = extract_imu_data(video_path_1) + accel_data_np = np.array(accel_data) + + accel_data_np[:, 0] = accel_data_np[:, 0] - accel_data_np[0, 0] + start_index = (np.abs(accel_data_np[:, 0] - start_clap_1)).argmin() + end_index = (np.abs(accel_data_np[:, 0] - end_clap_1)).argmin() + + gyro_data_np = np.array(gyro_data) + gyro_data_np[:, 0] = gyro_data_np[:, 0] - gyro_data_np[0, 0] + gyro_start_index = (np.abs(gyro_data_np[:, 0] - start_clap_1)).argmin() + gyro_end_index = (np.abs(gyro_data_np[:, 0] - end_clap_1)).argmin() + + imu_data_json = {} + imu_data_json['accel'] = accel_data_np[start_index:end_index, :].tolist() + imu_data_json['gyro'] = gyro_data_np[gyro_start_index:gyro_end_index, :].tolist() + + with open(os.path.join(args.input_dir, "imu.json"), "w", encoding="UTF-8") as file: + json.dump(imu_data_json, file, indent=4) + trim_video(video_path_1, start_clap_1, end_clap_1, output_video_path_1) trim_video(video_path_2, start_clap_2, end_clap_2, output_video_path_2) if round(get_video_length(output_video_path_1)) != round(get_video_length(output_video_path_2)): + print("clap_times_1: ", clap_times_1, "clap_times_2: ", clap_times_2) raise ValueError("The videos are not the same lenght.") os.remove(video_path_1)