From da480f795ccc0cd39d84ef8e45c8b3d2055eabef Mon Sep 17 00:00:00 2001 From: dotag1 Date: Mon, 8 Jul 2024 14:43:06 -0400 Subject: [PATCH] New scripts for processing the videos --- README-pandora.md | 163 +++++++++--------- lantern_scripts/create_data_folder.py | 74 ++++++++ lantern_scripts/create_sfm_folder.py | 60 +++++++ lantern_scripts/mp4_videos_to_png_frames.py | 6 +- lantern_scripts/process_videos_for_sfm.py | 14 ++ lantern_scripts/sync_and_trim.py | 105 +++++++++++ .../data/datamanagers/base_datamanager.py | 1 + nerfstudio/lantern/model.py | 4 +- nerfstudio/models/nerfacto.py | 1 + 9 files changed, 338 insertions(+), 90 deletions(-) create mode 100644 lantern_scripts/create_data_folder.py create mode 100644 lantern_scripts/create_sfm_folder.py create mode 100644 lantern_scripts/process_videos_for_sfm.py create mode 100644 lantern_scripts/sync_and_trim.py diff --git a/README-pandora.md b/README-pandora.md index 89cb052cc..933ce0807 100644 --- a/README-pandora.md +++ b/README-pandora.md @@ -1,5 +1,3 @@ - - # How it works? ## 0. Setup @@ -11,128 +9,123 @@ pip install skylibs equilib ## 1. Data acquisition & pre-processing -### OPTION 1: Real data - 1. Acquire data using the apparatus (stick with two Ricoh cameras) Make sure the cameras point up to avoid having lights in the seam. - Do two recordings with both Ricoh Theta cameras. Use the flash of a flashlight both times, the falling edge will later be detected by a software. + Do two recordings with both Ricoh Theta cameras. + + Camera setup: + Install Ricoh Theta app + Connect the camera to the app + Change camera parameters: select manual (bottom right) + First time: turn on "CT Settings" in Shooting settings + Apreture: 2.1 + Shutter speed: determined below + ISO: 800 + WB: 3500 + + To connect the other camera: + Turn wifi off + Close the app + Turn wifi on + Open the app + Connect the other camera > Cameras: > > **camera 1**: left, well-exposed, serial: YN14100695 \ > **camera 3**: right, under-exposed, serial: YN14111000 -2. Take one video capture with the two cameras well-exposed. +2. Take one video capture with the two cameras well-exposed. IMPORTANT: Note the shutter speed!!! - Use flash at the beginning (and at the end?) to use for synchronization. + Change shutter speed to get a well exposed image. + Clap at the beginning and at the end to use for synchronization. 3. Take one video capture with Camera 1 (left) well-exposed and Camera 2 (right) fast-exposed. - - Use flash at the beginning (and at the end?) to use for synchronization. - -4. Run the script for automatic flash detection / use Shotcut to manually align and trim the videos manually. - It will temporally align the frames from the left and right camera and remove the initial frames with the flash. + Change shutter speed of Camera 2 to 1/25000 (fastest exposure). + Clap at the beginning and at the end to use for synchronization. -5. Extract frames using ffmpeg +4. Import the files on computer + + If on mac, use Ricoh Theta File Transfer for Mac + +5. Get the equirectangular videos by processing them with the Ricoh Theta computer app. + + Drag and drop the video file into the app window. + Make sure both boxes are unchecked. + Name files as fallows + Camera 1: + First capture: left_sfm + Second capture: left_e1 + Camera 2: + First capture: right_sfm + Second capture: right_e2 + +6. Transfer videos on lab machine. ``` - python lantern_scripts/mp4_videos_to_png_frames.py --help - python lantern_scripts/mp4_videos_to_png_frames.py --input_dir data/lab_downstairs/trimmed_videos + scp -r /path/to/source/file /path/to/destination/ ``` - It should output the following file structure - ``` - left_colmap_baseline/ - ....png - ....png - ....png - right_colmap_baseline/ - ....png - ....png - ....png - left_e1/ - ....png - ....png - ....png - right_e2/ - ....png - ....png - ....png - ``` - -6. Run the code to mask out people and the stick. +7. Process the videos for OpenSFM. - Make sure the masks in "./stick_masks" are valid ``` - python lantern_scripts/mask_humans2.py --help - python lantern_scripts/mask_humans2.py --input_dir data/lab_downstairs/trimmed_videos --output_dir + python lantern_scripts/process_videos_for_sfm.py --input_dir /mnt/data/small_scene_window/ --shutter_speed 0.004 ``` - -### OPTION 2: Synthetic data - -`` -export PYTHONPATH=$pwd:$PYTHONPATH -python lantern_scripts/split_synthetic_colmap.py --has_exposure --left_dir ./Dual_Cameras/left --right_dir ./Dual_Cameras/right --out_dir ./Dual_Cameras/split_colmap_out -`` - -## 2. Process data using COLMAP - -Example with synthetic data: +## 2. Process data using OpenSFM ``` -export PYTHONPATH=$pwd:$PYTHONPATH -ns-process-data lantern --data ./Dual_Cameras/split_colmap_subset --output-dir ./Dual_Cameras/split_colmap_subset/split_colmap_out5 --skip-image-processing --camera-type equirectangular --images-per-equirect 8 +bin/opensfm_run_all /mnt/data/scene/sfm/ ``` -Example with real data (note that `--mask-dir` was added): +## 2. Process data for NeRF ``` -export PYTHONPATH=$pwd:$PYTHONPATH -ns-process-data lantern --data ./data/lab_downstairs/captured_data --mask-dir ./data/lab_downstairs/masks --output-dir ./data/lab_downstairs/colmap_out --skip-image-processing --camera-type equirectangular --images-per-equirect 8 +ns-process-data lantern-openSFM --data /mnt/data/small_scene/data/ --output-dir /mnt/data/small_scene/ns/ --metadata /mnt/data/small_scene/sfm/reconstruction.json ``` -After that, you'll need to copy manually (for now), the folder to `images`, for nerf training. +## 3. Train NeRF -Examples +1. Run step 1 of lantern-nerfacto. + + ``` + ns-train lantern-nerfacto --data /mnt/data/meeting_room_small_ns_no_exr/ --viewer.websocket-port 8008 --pipeline.datamanager.train-num-images-to-sample-from 1800 --pipeline.model.lantern_steps 1 --pipeline.datamanager.pixel-sampler.lantern_steps 1 --max-num-iterations 60000 + ``` + +2. Run step 2 of lantern-nerfacto. + + ``` + ns-train lantern-nerfacto --data /mnt/data/meeting_room_small_ns/ --viewer.websocket-port 8008 --pipeline.datamanager.train-num-images-to-sample-from 1800 --pipeline.model.lantern_steps 2 --pipeline.datamanager.pixel-sampler.lantern_steps 2 --load-dir /mnt/workspace/lantern/nerfstudio-HDR/outputs/meeting_room_small_ns/lantern-nerfacto/2024-06-10_155559/nerfstudio_models --pipeline.model.apply_mu_law False + ``` + +## 4. View results + +Use the interface : ``` -ns-process-data lantern --data ./data/lab_ground_floor/trimmed_videos_smart_subset --mask-dir ./data/lab_ground_floor/trimmed_videos_masks --output-dir ./data/lab_ground_floor/colmap_out_smart --skip-image-processing --camera-type equirectangular --images-per --equirect 8 --skip-colmap --skip-perspective-transform +ns-viewer --load-config outputs/tree_hill/nerfacto/2024-06-06_191338/config.yml --viewer.websocket-port 8008 ``` -## 2. Pre-training NeRF - -Using both the well-exposed and fast-exposed frames, as well as their associated camera poses, a NeRF will be trained to predict both the RGB and mask to do volumetric rendering. - -After convergence, go to the next step. - +Render a video with the interface command: +Move around and add cameras to manualy set a camera path. ``` -ns-train lantern-nerfacto --help +ns-render camera-path --load-config outputs/tree_hill/nerfacto/2024-06-06_191338/cclearonfig.yml --camera-path-filename /mnt/data/tree_hill/camera_paths/2024-06-06_191338.json --output-path renders/tree_hill/2024-06-06_191338.mp4 ``` -If you need to use less frames, add this argument: `--pipeline.datamanager.train-num-images-to-sample-from 200 --pipeline.datamanager.eval-num-images-to-sample-from 30` - -> Thing to keep in mind: the way Nerfstudio splits train and test images may not be optimal: left might be in one dataset and right in another... that's a bit of a data leak. - -Examples - +Get evaluation images: ``` -ns-train lantern-nerfacto --data data/lab_ground_floor/colmap_out_smart --pipeline.datamanager.train-num-images-to-sample-from 200 --pipeline.datamanager.eval-num-images-to-sample-from 30 +ns-eval --load-config=outputs/meeting_room_ns_no_exr/nerfacto/2024-06-17_152246/config.yml --output-path=output.json --render_output_path=/mnt/data/temp_folder_meeting_room_unaligned ``` -## 3. Running MomoNet - -(Using which frames?) A pretrained UNet image-to-image network called MomoNet will infill the missing dynamic range and store these predictions in new images. -It will be trained on the Laval indoor HDR dataset to do this task. - -TODO: validate this part - -## 4. Train from scratch or fine-tune the NeRF model using MomoNet's predictions - - - +To get evaluation images and tensorboard: ``` -``` \ No newline at end of file +ns-eval --load-config=outputs/meeting_room_ns_no_exr/lantern-nerfacto/2024-06-20_142413/config.yml --output-path=output.json --render_output_path=/mnt/data/temp_folder_meeting_room_large_dataset_2 --vis viewer+tensorboard +``` + +Load tensorboard results: +``` +tensorboard --logdir=Downloads/images_for_pts/test/runs/ +``` +And connect to http://localhost:6006/ diff --git a/lantern_scripts/create_data_folder.py b/lantern_scripts/create_data_folder.py new file mode 100644 index 000000000..95cc7d19a --- /dev/null +++ b/lantern_scripts/create_data_folder.py @@ -0,0 +1,74 @@ +import os +import shutil +import json +import argparse + +def move_subfolder(source_dir, destination_dir): + if not os.path.exists(source_dir): + print(f"Source directory '{source_dir}' does not exist.") + return + + os.makedirs(destination_dir, exist_ok=True) + + try: + for filename in os.listdir(source_dir): + if filename.lower().count('png') < 2: + shutil.move(os.path.join(source_dir, filename), os.path.join(destination_dir, filename)) + except Exception as e: + print(f"Error moving subfolder: {e}") + + delete_empty_folder(source_dir) + +def delete_empty_folder(folder_path): + try: + if os.path.exists(folder_path): + if not os.listdir(folder_path): + os.rmdir(folder_path) + print(f"Folder '{folder_path}' deleted successfully.") + else: + print(f"Folder '{folder_path}' does not exist.") + except Exception as e: + print(f"Error deleting folder: {e}") + +def create_capture_settings_file(file_path, shutter_speed): + capture_settings = { + "left": { + "white_balance": 3500, + "shutter_speed": shutter_speed, + "iso": 800 + }, + "right": { + "white_balance": 3500, + "shutter_speed": 0.00004, + "iso": 800 + } + } + + try: + with open(file_path, 'w') as file: + json.dump(capture_settings, file, indent=4) + except Exception as e: + print(f"Error creating JSON file: {e}") + +if __name__ == "__main__": + argparser = argparse.ArgumentParser() + argparser.add_argument("--input_dir", type=str, default="/mnt/data/scene/") + argparser.add_argument("--shutter_speed", type=float, default=0.003125) + args = argparser.parse_args() + + dest_folder = args.input_dir + masks_folder = args.input_dir + "masks/" + + for item in os.listdir(masks_folder): + move_subfolder(masks_folder + item, dest_folder + item + "/mask") + + delete_empty_folder(args.input_dir + "masks") + + create_capture_settings_file(dest_folder + "capture_settings.json", args.shutter_speed) + + + + + + + diff --git a/lantern_scripts/create_sfm_folder.py b/lantern_scripts/create_sfm_folder.py new file mode 100644 index 000000000..73802e535 --- /dev/null +++ b/lantern_scripts/create_sfm_folder.py @@ -0,0 +1,60 @@ +import os +import shutil +import json +import yaml +import argparse + +def copy_images(src_folder, dest_folder): + os.makedirs(dest_folder, exist_ok=True) + + files = os.listdir(src_folder) + for filename in files: + if filename.endswith(".png"): + src_file = os.path.join(src_folder, filename) + dest_file = os.path.join(dest_folder, filename) + shutil.copy(src_file, dest_file) + +def create_camera_models_overrides_json(dest_folder): + camera_models_data = { + "all": { + "width": 3840, + "projection_type": "equirectangular", + "height": 1920 + } + } + + json_file_path = os.path.join(dest_folder, 'camera_models_overrides.json') + + with open(json_file_path, 'w') as json_file: + json.dump(camera_models_data, json_file, indent=2) + +def create_config_yml(dest_folder): + config_data = { + "processes": 8, + "read_processes": 8, + } + + yaml_file_path = os.path.join(dest_folder, 'config.yml') + + with open(yaml_file_path, 'w') as yaml_file: + yaml_file.write("# Other params\n") + yaml.dump(config_data, yaml_file, default_flow_style=False) + + +if __name__ == '__main__': + argparser = argparse.ArgumentParser() + argparser.add_argument("--input_dir", type=str, default="/mnt/data/scene/") + args = argparser.parse_args() + + src_folder = args.input_dir + "data/" + dest_folder = args.input_dir + "sfm/" + dest_image_folder = dest_folder + "images" + dest_mask_folder = dest_folder + "masks" + + for folder in ("left_e1/", "left_sfm/", "right_sfm/"): + copy_images(src_folder + folder, dest_image_folder) + copy_images(src_folder + folder + "mask/", dest_mask_folder) + + create_camera_models_overrides_json(dest_folder) + create_config_yml(dest_folder) + \ No newline at end of file diff --git a/lantern_scripts/mp4_videos_to_png_frames.py b/lantern_scripts/mp4_videos_to_png_frames.py index b675ca4b4..e4246f12b 100644 --- a/lantern_scripts/mp4_videos_to_png_frames.py +++ b/lantern_scripts/mp4_videos_to_png_frames.py @@ -30,14 +30,14 @@ if __name__ == '__main__': "ffmpeg", "-i", video, "-vf", "fps=4.0", - f"{output_folder}/output_%04d.png" + f"{output_folder}/{video_name}_output_%04d.png" ] else: command = [ "ffmpeg", "-i", video, "-vf", "fps=15.0", - f"{output_folder}/output_%04d.png" + f"{output_folder}/{video_name}_output_%04d.png" ] subprocess.run(command, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) @@ -45,7 +45,7 @@ if __name__ == '__main__': for file in os.listdir(output_folder): if not file.endswith(".png"): continue - frame_number = int(os.path.splitext(file)[0].split("_")[1]) + frame_number = int(os.path.splitext(file)[0].split("_")[-1]) if frame_number % 10 != 0: os.remove(os.path.join(output_folder, file)) diff --git a/lantern_scripts/process_videos_for_sfm.py b/lantern_scripts/process_videos_for_sfm.py new file mode 100644 index 000000000..f4bda5575 --- /dev/null +++ b/lantern_scripts/process_videos_for_sfm.py @@ -0,0 +1,14 @@ +import subprocess +import argparse + +if __name__ == '__main__': + argparser = argparse.ArgumentParser() + argparser.add_argument("--input_dir", type=str, required=True, default="/mnt/data/scene/", help="Directory containing all the 4 videos") + argparser.add_argument("--shutter_speed", type=float, required=True, default=0.003125) + args = argparser.parse_args() + + subprocess.run(["python", "lantern_scripts/sync_and_trim.py", "--input_dir", args.input_dir]) + subprocess.run(["python", "lantern_scripts/mp4_videos_to_png_frames.py", "--input_dir", args.input_dir + "data/"]) + subprocess.run(["python", "lantern_scripts/mask_humans2.py", "--input_dir", args.input_dir + "data/", "--output_dir", args.input_dir + "data/masks"]) + subprocess.run(["python", "lantern_scripts/create_data_folder.py", "--input_dir", args.input_dir + "data/", "--shutter_speed", str(args.shutter_speed)]) + subprocess.run(["python", "lantern_scripts/create_sfm_folder.py", "--input_dir", args.input_dir]) diff --git a/lantern_scripts/sync_and_trim.py b/lantern_scripts/sync_and_trim.py new file mode 100644 index 000000000..4f6d0b975 --- /dev/null +++ b/lantern_scripts/sync_and_trim.py @@ -0,0 +1,105 @@ +import argparse +import os +import subprocess +from pydub import AudioSegment +import numpy as np +from scipy.signal import find_peaks + +def extract_audio(video_path, audio_path): + command = [ + "ffmpeg", + "-i", video_path, + "-q:a", "0", + "-map", "a", + audio_path + ] + subprocess.run(command) + +def detect_hand_claps(audio_path, threshold=0.5, min_distance=10000): + audio = AudioSegment.from_wav(audio_path) + samples = np.array(audio.get_array_of_samples()) + + # Normalize the audio signal + samples = samples / np.max(np.abs(samples)) + + # Find peaks + peaks, _ = find_peaks(samples, height=threshold, distance=min_distance) + + # Convert sample indices to time in seconds + clap_times = [p / audio.frame_rate for p in peaks] + + return clap_times + +def trim_video(video_path, start_time, end_time, output_path): + command = [ + "ffmpeg", + "-i", video_path, + "-ss", str(start_time), + "-to", str(end_time), + "-c", "copy", + output_path + ] + subprocess.run(command) + +def get_video_length(video_path): + command = [ + "ffprobe", + "-v", "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + video_path + ] + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + return float(result.stdout.strip()) + +if __name__ == '__main__': + argparser = argparse.ArgumentParser() + argparser.add_argument("--input_dir", type=str, default="/mnt/data/scene/", help="Directory containing all the 4 videos") + args = argparser.parse_args() + + os.makedirs(os.path.join(args.input_dir, "data"), exist_ok=True) + + for i in range(2): + if i == 0: + video_path_1 = os.path.join(args.input_dir, "left_e1_er.MP4") + video_path_2 = os.path.join(args.input_dir, "right_e2_er.MP4") + audio_path_1 = os.path.join(args.input_dir, "left_e1_audio.wav") + audio_path_2 = os.path.join(args.input_dir, "right_e2_audio.wav") + output_video_path_1 = os.path.join(args.input_dir, "data/left_e1.mp4") + output_video_path_2 = os.path.join(args.input_dir, "data/right_e2.mp4") + + if i == 1: + video_path_1 = os.path.join(args.input_dir, "left_sfm_er.MP4") + video_path_2 = os.path.join(args.input_dir, "right_sfm_er.MP4") + audio_path_1 = os.path.join(args.input_dir, "left_sfm_audio.wav") + audio_path_2 = os.path.join(args.input_dir, "right_sfm_audio.wav") + output_video_path_1 = os.path.join(args.input_dir, "data/left_sfm.mp4") + output_video_path_2 = os.path.join(args.input_dir, "data/right_sfm.mp4") + + extract_audio(video_path_1, audio_path_1) + extract_audio(video_path_2, audio_path_2) + + clap_times_1 = detect_hand_claps(audio_path_1) + clap_times_2 = detect_hand_claps(audio_path_2) + + if len(clap_times_1) < 2 or len(clap_times_2) < 2: + raise ValueError("Not enough hand claps detected in one of the audios.") + + start_clap_1, end_clap_1 = clap_times_1[0], clap_times_1[-1] + start_clap_2, end_clap_2 = clap_times_2[0], clap_times_2[-1] + + trim_video(video_path_1, start_clap_1, end_clap_1, output_video_path_1) + trim_video(video_path_2, start_clap_2, end_clap_2, output_video_path_2) + + if round(get_video_length(output_video_path_1)) != round(get_video_length(output_video_path_2)): + raise ValueError("The videos are not the same lenght.") + + os.remove(video_path_1) + os.remove(video_path_2) + os.remove(audio_path_1) + os.remove(audio_path_2) + + print(f"Left video trimmed from {start_clap_1} to {end_clap_1}, length : {get_video_length(output_video_path_1)}") + print(f"Right video trimmed from {start_clap_2} to {end_clap_2}, length : {get_video_length(output_video_path_2)}") diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index 0d73c0459..5a373d491 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -497,6 +497,7 @@ class VanillaDataManager(DataManager, Generic[TDataset]): """Sets up the data loader for evaluation""" assert self.eval_dataset is not None CONSOLE.print("Setting up evaluation dataset...") + # self.eval_dataset = self.train_dataset # for testing alignment issue self.eval_image_dataloader = CacheDataloader( self.eval_dataset, num_images_to_sample_from=self.config.eval_num_images_to_sample_from, diff --git a/nerfstudio/lantern/model.py b/nerfstudio/lantern/model.py index f522bc02f..4ac99b229 100644 --- a/nerfstudio/lantern/model.py +++ b/nerfstudio/lantern/model.py @@ -235,11 +235,11 @@ class LanternModel(NerfactoModel): ) -> Tuple[Dict[str, float], Dict[str, torch.Tensor]]: gt_rgb = batch["image"].to(self.device) - if batch["exposure"] != 1.0: + if batch["exposure"] != 1.0: # change to == for testing fast-exposed predicted_rgb = outputs["rgb"] # Blended with background (black if random background) else: # if self.config.lantern_steps == 1: - return None, None + return None, None # comment for testing fast-exposed predicted_rgb = outputs["rgb_fast"] # Blended with background (black if random background) # if self.config.apply_mu_law: # from nerfstudio.utils.colormaps import ColormapOptions, Colormaps diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py index 1d243632b..b4d36ccc9 100644 --- a/nerfstudio/models/nerfacto.py +++ b/nerfstudio/models/nerfacto.py @@ -373,6 +373,7 @@ class NerfactoModel(Model): ) combined_rgb = torch.cat([gt_rgb, predicted_rgb], dim=1) + # combined_rgb = predicted_rgb # for simple output without GT combined_acc = torch.cat([acc], dim=1) combined_depth = torch.cat([depth], dim=1)