New scripts for processing the videos

lantern_scripts/create_data_folder.py

@@ -0,0 +1,74 @@
+import os
+import shutil
+import json
+import argparse
+
+def move_subfolder(source_dir, destination_dir):
+    if not os.path.exists(source_dir):
+        print(f"Source directory '{source_dir}' does not exist.")
+        return
+    
+    os.makedirs(destination_dir, exist_ok=True)
+
+    try:
+        for filename in os.listdir(source_dir):
+            if filename.lower().count('png') < 2:
+                shutil.move(os.path.join(source_dir, filename), os.path.join(destination_dir, filename))
+    except Exception as e:
+        print(f"Error moving subfolder: {e}")
+
+    delete_empty_folder(source_dir)
+
+def delete_empty_folder(folder_path):
+    try:
+        if os.path.exists(folder_path):
+            if not os.listdir(folder_path):
+                os.rmdir(folder_path)
+                print(f"Folder '{folder_path}' deleted successfully.")
+        else:
+            print(f"Folder '{folder_path}' does not exist.")
+    except Exception as e:
+        print(f"Error deleting folder: {e}")
+
+def create_capture_settings_file(file_path, shutter_speed):
+    capture_settings = {
+        "left": {
+            "white_balance": 3500,
+            "shutter_speed": shutter_speed,
+            "iso": 800
+        },
+        "right": {
+            "white_balance": 3500,
+            "shutter_speed": 0.00004,
+            "iso": 800
+        }
+    }
+
+    try:
+        with open(file_path, 'w') as file:
+            json.dump(capture_settings, file, indent=4)
+    except Exception as e:
+        print(f"Error creating JSON file: {e}")
+
+if __name__ == "__main__":
+    argparser = argparse.ArgumentParser()
+    argparser.add_argument("--input_dir", type=str, default="/mnt/data/scene/")
+    argparser.add_argument("--shutter_speed", type=float, default=0.003125)
+    args = argparser.parse_args()
+
+    dest_folder = args.input_dir
+    masks_folder = args.input_dir + "masks/"
+
+    for item in os.listdir(masks_folder):
+        move_subfolder(masks_folder + item, dest_folder + item + "/mask")
+
+    delete_empty_folder(args.input_dir + "masks")
+
+    create_capture_settings_file(dest_folder + "capture_settings.json", args.shutter_speed)
+
+
+
+
+    
+
+

lantern_scripts/create_sfm_folder.py

@@ -0,0 +1,60 @@
+import os
+import shutil
+import json
+import yaml
+import argparse
+
+def copy_images(src_folder, dest_folder):
+    os.makedirs(dest_folder, exist_ok=True)
+
+    files = os.listdir(src_folder)
+    for filename in files:
+        if filename.endswith(".png"):
+            src_file = os.path.join(src_folder, filename)
+            dest_file = os.path.join(dest_folder, filename)
+            shutil.copy(src_file, dest_file)
+
+def create_camera_models_overrides_json(dest_folder):
+    camera_models_data = {
+        "all": {
+            "width": 3840,
+            "projection_type": "equirectangular",
+            "height": 1920
+        }
+    }
+
+    json_file_path = os.path.join(dest_folder, 'camera_models_overrides.json')
+    
+    with open(json_file_path, 'w') as json_file:
+        json.dump(camera_models_data, json_file, indent=2)
+
+def create_config_yml(dest_folder):
+    config_data = {
+        "processes": 8,
+        "read_processes": 8,
+    }
+    
+    yaml_file_path = os.path.join(dest_folder, 'config.yml')
+    
+    with open(yaml_file_path, 'w') as yaml_file:
+        yaml_file.write("# Other params\n")
+        yaml.dump(config_data, yaml_file, default_flow_style=False)
+
+
+if __name__ == '__main__':
+    argparser = argparse.ArgumentParser()
+    argparser.add_argument("--input_dir", type=str, default="/mnt/data/scene/")
+    args = argparser.parse_args()
+
+    src_folder = args.input_dir + "data/"
+    dest_folder = args.input_dir + "sfm/"
+    dest_image_folder = dest_folder + "images"
+    dest_mask_folder = dest_folder + "masks"
+
+    for folder in ("left_e1/", "left_sfm/", "right_sfm/"):
+        copy_images(src_folder + folder, dest_image_folder)
+        copy_images(src_folder + folder + "mask/", dest_mask_folder)
+
+    create_camera_models_overrides_json(dest_folder)
+    create_config_yml(dest_folder)
+    

lantern_scripts/mp4_videos_to_png_frames.py

@@ -30,14 +30,14 @@ if __name__ == '__main__':
                 "ffmpeg",
                 "-i", video,
                 "-vf", "fps=4.0",
-                f"{output_folder}/output_%04d.png"
+                f"{output_folder}/{video_name}_output_%04d.png"
             ]
         else:
             command = [
                 "ffmpeg",
                 "-i", video,
                 "-vf", "fps=15.0",
-                f"{output_folder}/output_%04d.png"
+                f"{output_folder}/{video_name}_output_%04d.png"
             ]
         subprocess.run(command, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
 
@@ -45,7 +45,7 @@ if __name__ == '__main__':
         for file in os.listdir(output_folder):
             if not file.endswith(".png"):
                 continue
-            frame_number = int(os.path.splitext(file)[0].split("_")[1])
+            frame_number = int(os.path.splitext(file)[0].split("_")[-1])
             if frame_number % 10 != 0:
                 os.remove(os.path.join(output_folder, file))
 

lantern_scripts/process_videos_for_sfm.py

@@ -0,0 +1,14 @@
+import subprocess
+import argparse
+
+if __name__ == '__main__':
+    argparser = argparse.ArgumentParser()
+    argparser.add_argument("--input_dir", type=str, required=True, default="/mnt/data/scene/", help="Directory containing all the 4 videos")
+    argparser.add_argument("--shutter_speed", type=float, required=True, default=0.003125)
+    args = argparser.parse_args()
+
+    subprocess.run(["python", "lantern_scripts/sync_and_trim.py", "--input_dir", args.input_dir])
+    subprocess.run(["python", "lantern_scripts/mp4_videos_to_png_frames.py", "--input_dir", args.input_dir + "data/"])
+    subprocess.run(["python", "lantern_scripts/mask_humans2.py", "--input_dir", args.input_dir + "data/", "--output_dir", args.input_dir + "data/masks"])
+    subprocess.run(["python", "lantern_scripts/create_data_folder.py", "--input_dir", args.input_dir + "data/", "--shutter_speed", str(args.shutter_speed)])
+    subprocess.run(["python", "lantern_scripts/create_sfm_folder.py", "--input_dir", args.input_dir])

lantern_scripts/sync_and_trim.py

@@ -0,0 +1,105 @@
+import argparse
+import os
+import subprocess
+from pydub import AudioSegment
+import numpy as np
+from scipy.signal import find_peaks
+
+def extract_audio(video_path, audio_path):
+    command = [
+        "ffmpeg",
+        "-i", video_path,
+        "-q:a", "0",
+        "-map", "a",
+        audio_path
+    ]
+    subprocess.run(command)
+
+def detect_hand_claps(audio_path, threshold=0.5, min_distance=10000):
+    audio = AudioSegment.from_wav(audio_path)
+    samples = np.array(audio.get_array_of_samples())
+    
+    # Normalize the audio signal
+    samples = samples / np.max(np.abs(samples))
+    
+    # Find peaks
+    peaks, _ = find_peaks(samples, height=threshold, distance=min_distance)
+    
+    # Convert sample indices to time in seconds
+    clap_times = [p / audio.frame_rate for p in peaks]
+    
+    return clap_times
+
+def trim_video(video_path, start_time, end_time, output_path):
+    command = [
+        "ffmpeg",
+        "-i", video_path,
+        "-ss", str(start_time),
+        "-to", str(end_time),
+        "-c", "copy",
+        output_path
+    ]
+    subprocess.run(command)
+
+def get_video_length(video_path):
+    command = [
+        "ffprobe",
+        "-v", "error",
+        "-show_entries",
+        "format=duration",
+        "-of",
+        "default=noprint_wrappers=1:nokey=1",
+        video_path
+    ]
+    result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+    return float(result.stdout.strip())
+
+if __name__ == '__main__':
+    argparser = argparse.ArgumentParser()
+    argparser.add_argument("--input_dir", type=str, default="/mnt/data/scene/", help="Directory containing all the 4 videos")
+    args = argparser.parse_args()
+
+    os.makedirs(os.path.join(args.input_dir, "data"), exist_ok=True)
+
+    for i in range(2):
+        if i == 0:
+            video_path_1 = os.path.join(args.input_dir, "left_e1_er.MP4")
+            video_path_2 = os.path.join(args.input_dir, "right_e2_er.MP4")
+            audio_path_1 = os.path.join(args.input_dir, "left_e1_audio.wav")
+            audio_path_2 = os.path.join(args.input_dir, "right_e2_audio.wav")
+            output_video_path_1 = os.path.join(args.input_dir, "data/left_e1.mp4")
+            output_video_path_2 = os.path.join(args.input_dir, "data/right_e2.mp4")
+
+        if i == 1:
+            video_path_1 = os.path.join(args.input_dir, "left_sfm_er.MP4")
+            video_path_2 = os.path.join(args.input_dir, "right_sfm_er.MP4")
+            audio_path_1 = os.path.join(args.input_dir, "left_sfm_audio.wav")
+            audio_path_2 = os.path.join(args.input_dir, "right_sfm_audio.wav")
+            output_video_path_1 = os.path.join(args.input_dir, "data/left_sfm.mp4")
+            output_video_path_2 = os.path.join(args.input_dir, "data/right_sfm.mp4")
+
+        extract_audio(video_path_1, audio_path_1)
+        extract_audio(video_path_2, audio_path_2)
+
+        clap_times_1 = detect_hand_claps(audio_path_1)
+        clap_times_2 = detect_hand_claps(audio_path_2)
+
+        if len(clap_times_1) < 2 or len(clap_times_2) < 2:
+            raise ValueError("Not enough hand claps detected in one of the audios.")
+
+        start_clap_1, end_clap_1 = clap_times_1[0], clap_times_1[-1]
+        start_clap_2, end_clap_2 = clap_times_2[0], clap_times_2[-1]
+
+        trim_video(video_path_1, start_clap_1, end_clap_1, output_video_path_1)
+        trim_video(video_path_2, start_clap_2, end_clap_2, output_video_path_2)
+
+        if round(get_video_length(output_video_path_1)) != round(get_video_length(output_video_path_2)):
+            raise ValueError("The videos are not the same lenght.")
+
+        os.remove(video_path_1)
+        os.remove(video_path_2)
+        os.remove(audio_path_1)
+        os.remove(audio_path_2)
+
+        print(f"Left video trimmed from {start_clap_1} to {end_clap_1}, length : {get_video_length(output_video_path_1)}")
+        print(f"Right video trimmed from {start_clap_2} to {end_clap_2}, length : {get_video_length(output_video_path_2)}")