diff --git a/nerfstudio/process_data/images_to_nerfstudio_dataset.py b/nerfstudio/process_data/images_to_nerfstudio_dataset.py index 4c0a5bf03..dc613412a 100644 --- a/nerfstudio/process_data/images_to_nerfstudio_dataset.py +++ b/nerfstudio/process_data/images_to_nerfstudio_dataset.py @@ -60,16 +60,24 @@ class ImagesToNerfstudioDataset(ColmapConverterToNerfstudioDataset): if not self.skip_image_processing: # Copy images to output directory image_rename_map_paths = process_data_utils.copy_images( - self.data, image_dir=self.image_dir, crop_factor=self.crop_factor, verbose=self.verbose + self.data, + image_dir=self.image_dir, + crop_factor=self.crop_factor, + verbose=self.verbose, + num_downscales=self.num_downscales, ) image_rename_map = dict((a.name, b.name) for a, b in image_rename_map_paths.items()) num_frames = len(image_rename_map) summary_log.append(f"Starting with {num_frames} images") - # Downscale images - summary_log.append( - process_data_utils.downscale_images(self.image_dir, self.num_downscales, verbose=self.verbose) + # # Create mask + mask_path = process_data_utils.save_mask( + image_dir=self.image_dir, + num_downscales=self.num_downscales, + crop_factor=(0.0, 0.0, 0.0, 0.0), ) + if mask_path is not None: + summary_log.append("Saved mask(s)") else: num_frames = len(process_data_utils.list_images(self.data)) if num_frames == 0: diff --git a/nerfstudio/process_data/polycam_utils.py b/nerfstudio/process_data/polycam_utils.py index 416da15ad..0b1ba7a62 100644 --- a/nerfstudio/process_data/polycam_utils.py +++ b/nerfstudio/process_data/polycam_utils.py @@ -131,6 +131,7 @@ def process_images( image_dir=image_dir, crop_border_pixels=crop_border_pixels, verbose=verbose, + num_downscales=num_downscales, ) num_frames = len(copied_image_paths) @@ -145,9 +146,6 @@ def process_images( else: summary_log.append(f"Started with {num_frames} images") - # Downscale images - summary_log.append(process_data_utils.downscale_images(image_dir, num_downscales, verbose=verbose)) - # Save json if num_frames == 0: CONSOLE.print("[bold red]No images found, exiting") @@ -190,7 +188,11 @@ def process_depth_maps( # Copy depth images to output directory copied_depth_maps_paths = process_data_utils.copy_and_upscale_polycam_depth_maps_list( - polycam_depth_maps_filenames, depth_dir=depth_dir, crop_border_pixels=crop_border_pixels, verbose=verbose + polycam_depth_maps_filenames, + depth_dir=depth_dir, + num_downscales=num_downscales, + crop_border_pixels=crop_border_pixels, + verbose=verbose, ) num_processed_depth_maps = len(copied_depth_maps_paths) @@ -211,11 +213,4 @@ def process_depth_maps( else: summary_log.append(f"Started with {num_processed_depth_maps} images") - # Downscale depth maps - summary_log.append( - process_data_utils.downscale_images( - depth_dir, num_downscales, folder_name="depths", nearest_neighbor=True, verbose=verbose - ) - ) - return summary_log, polycam_depth_maps_filenames diff --git a/nerfstudio/process_data/process_data_utils.py b/nerfstudio/process_data/process_data_utils.py index 9bd835c3a..6d4cee094 100644 --- a/nerfstudio/process_data/process_data_utils.py +++ b/nerfstudio/process_data/process_data_utils.py @@ -111,6 +111,7 @@ def convert_video_to_images( video_path: Path, image_dir: Path, num_frames_target: int, + num_downscales: int, crop_factor: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0), verbose: bool = False, ) -> Tuple[List[str], int]: @@ -120,6 +121,7 @@ def convert_video_to_images( video_path: Path to the video. output_dir: Path to the output directory. num_frames_target: Number of frames to extract. + num_downscales: Number of times to downscale the images. Downscales by 2 each time. crop_factor: Portion of the image to crop. Should be in [0,1] (top, bottom, left, right) verbose: If True, logs the output of the command. Returns: @@ -151,7 +153,6 @@ def convert_video_to_images( sys.exit(1) CONSOLE.print("Number of frames in video:", num_frames) - out_filename = image_dir / "frame_%05d.png" ffmpeg_cmd = f'ffmpeg -i "{video_path}"' crop_cmd = "" @@ -160,41 +161,66 @@ def convert_video_to_images( width = 1 - crop_factor[2] - crop_factor[3] start_x = crop_factor[2] start_y = crop_factor[0] - crop_cmd = f',"crop=w=iw*{width}:h=ih*{height}:x=iw*{start_x}:y=ih*{start_y}"' + crop_cmd = f"crop=w=iw*{width}:h=ih*{height}:x=iw*{start_x}:y=ih*{start_y}," + num_frames = get_num_frames_in_video(video_path) spacing = num_frames // num_frames_target + + downscale_chains = [f"[t{i}]scale=iw/{2**i}:ih/{2**i}[out{i}]" for i in range(num_downscales + 1)] + downscale_dirs = [Path(str(image_dir) + (f"_{2**i}" if i > 0 else "")) for i in range(num_downscales + 1)] + downscale_paths = [downscale_dirs[i] / "frame_%05d.png" for i in range(num_downscales + 1)] + + for dir in downscale_dirs: + dir.mkdir(parents=True, exist_ok=True) + + downscale_chain = ( + f"split={num_downscales + 1}" + + "".join([f"[t{i}]" for i in range(num_downscales + 1)]) + + ";" + + ";".join(downscale_chains) + ) + if spacing > 1: - ffmpeg_cmd += f" -vf thumbnail={spacing},setpts=N/TB{crop_cmd} -r 1" CONSOLE.print("Number of frames to extract:", math.ceil(num_frames / spacing)) + ffmpeg_cmd += " -vsync vfr -r1" + select_cmd = f"thumbnail={spacing},setpts=N/TB," else: CONSOLE.print("[bold red]Can't satisfy requested number of frames. Extracting all frames.") ffmpeg_cmd += " -pix_fmt bgr8" - if crop_cmd != "": - ffmpeg_cmd += f" -vf {crop_cmd[1:]}" + select_cmd = "" + + downscale_cmd = f' -filter_complex "{select_cmd}{crop_cmd}{downscale_chain}"' + "".join( + [f' -map "[out{i}]" "{downscale_paths[i]}"' for i in range(num_downscales + 1)] + ) + + ffmpeg_cmd += downscale_cmd - ffmpeg_cmd += f" {out_filename}" run_command(ffmpeg_cmd, verbose=verbose) - num_final_frames = len(list(image_dir.glob("*.png"))) - summary_log = [] - summary_log.append(f"Starting with {num_frames} video frames") - summary_log.append(f"We extracted {num_final_frames} images") - CONSOLE.log("[bold green]:tada: Done converting video to images.") + num_final_frames = len(list(image_dir.glob("*.png"))) + summary_log = [] + summary_log.append(f"Starting with {num_frames} video frames") + summary_log.append(f"We extracted {num_final_frames} images") + CONSOLE.log("[bold green]:tada: Done converting video to images.") - return summary_log, num_final_frames + return summary_log, num_final_frames def copy_images_list( image_paths: List[Path], image_dir: Path, + num_downscales: int, crop_border_pixels: Optional[int] = None, crop_factor: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0), verbose: bool = False, + upscale_factor: Optional[int] = None, + nearest_neighbor: bool = False, ) -> List[Path]: """Copy all images in a list of Paths. Useful for filtering from a directory. Args: image_paths: List of Paths of images to copy to a new directory. image_dir: Path to the output directory. + num_downscales: Number of times to downscale the images. Downscales by 2 each time. crop_border_pixels: If not None, crops each edge by the specified number of pixels. crop_factor: Portion of the image to crop. Should be in [0,1] (top, bottom, left, right) verbose: If True, print extra logging. @@ -230,26 +256,48 @@ def copy_images_list( pass copied_image_paths.append(copied_image_path) + nn_flag = "" if not nearest_neighbor else ":flags=neighbor" + downscale_chains = [f"[t{i}]scale=iw/{2**i}:ih/{2**i}{nn_flag}[out{i}]" for i in range(num_downscales + 1)] + downscale_dirs = [Path(str(image_dir) + (f"_{2**i}" if i > 0 else "")) for i in range(num_downscales + 1)] + downscale_paths = [ + downscale_dirs[i] / ("frame_%05d" + copied_image_paths[0].suffix) for i in range(num_downscales + 1) + ] + + for dir in downscale_dirs: + dir.mkdir(parents=True, exist_ok=True) + + downscale_chain = ( + f"split={num_downscales + 1}" + + "".join([f"[t{i}]" for i in range(num_downscales + 1)]) + + ";" + + ";".join(downscale_chains) + ) + + num_frames = len(image_paths) + ffmpeg_cmd = f'ffmpeg -y -noautorotate -i "{image_dir / f"frame_%05d{copied_image_paths[0].suffix}"}" -q:v 2 ' + + crop_cmd = "" if crop_border_pixels is not None: - file_type = image_paths[0].suffix - filename = f"frame_%05d{file_type}" - crop = f"crop=iw-{crop_border_pixels*2}:ih-{crop_border_pixels*2}" - ffmpeg_cmd = f'ffmpeg -y -noautorotate -i "{image_dir / filename}" -q:v 2 -vf {crop} "{image_dir / filename}"' - run_command(ffmpeg_cmd, verbose=verbose) + crop_cmd = f"crop=iw-{crop_border_pixels*2}:ih-{crop_border_pixels*2}[cropped];[cropped]" elif crop_factor != (0.0, 0.0, 0.0, 0.0): - file_type = image_paths[0].suffix - filename = f"frame_%05d{file_type}" height = 1 - crop_factor[0] - crop_factor[1] width = 1 - crop_factor[2] - crop_factor[3] start_x = crop_factor[2] start_y = crop_factor[0] - crop_cmd = f',"crop=w=iw*{width}:h=ih*{height}:x=iw*{start_x}:y=ih*{start_y}"' - ffmpeg_cmd = ( - f'ffmpeg -y -noautorotate -i "{image_dir / filename}" -q:v 2 -vf {crop_cmd[1:]} "{image_dir / filename}"' - ) - run_command(ffmpeg_cmd, verbose=verbose) + crop_cmd = f"crop=w=iw*{width}:h=ih*{height}:x=iw*{start_x}:y=ih*{start_y}[cropped];[cropped]" - num_frames = len(image_paths) + select_cmd = "[0:v]" + if upscale_factor is not None: + select_cmd = f"[0:v]scale=iw*{upscale_factor}:ih*{upscale_factor}:flags=neighbor[upscaled];[upscaled]" + + downscale_cmd = f' -filter_complex "{select_cmd}{crop_cmd}{downscale_chain}"' + "".join( + [f' -map "[out{i}]" "{downscale_paths[i]}"' for i in range(num_downscales + 1)] + ) + + ffmpeg_cmd += downscale_cmd + if verbose: + CONSOLE.log(f"... {ffmpeg_cmd}") + run_command(ffmpeg_cmd, verbose=verbose) if num_frames == 0: CONSOLE.log("[bold red]:skull: No usable images in the data folder.") @@ -262,6 +310,7 @@ def copy_images_list( def copy_and_upscale_polycam_depth_maps_list( polycam_depth_image_filenames: List[Path], depth_dir: Path, + num_downscales: int, crop_border_pixels: Optional[int] = None, verbose: bool = False, ) -> List[Path]: @@ -284,31 +333,26 @@ def copy_and_upscale_polycam_depth_maps_list( assert upscale_factor > 1 assert isinstance(upscale_factor, int) - copied_depth_map_paths = [] - for idx, depth_map in enumerate(polycam_depth_image_filenames): - destination = depth_dir / f"frame_{idx + 1:05d}{depth_map.suffix}" - ffmpeg_cmd = [ - f'ffmpeg -y -i "{depth_map}" ', - f"-q:v 2 -vf scale=iw*{upscale_factor}:ih*{upscale_factor}:flags=neighbor ", - f'"{destination}"', - ] - ffmpeg_cmd = " ".join(ffmpeg_cmd) - run_command(ffmpeg_cmd, verbose=verbose) - copied_depth_map_paths.append(destination) - - if crop_border_pixels is not None: - file_type = depth_dir.glob("frame_*").__next__().suffix - filename = f"frame_%05d{file_type}" - crop = f"crop=iw-{crop_border_pixels * 2}:ih-{crop_border_pixels * 2}" - ffmpeg_cmd = f'ffmpeg -y -i "{depth_dir / filename}" -q:v 2 -vf {crop} "{depth_dir / filename}"' - run_command(ffmpeg_cmd, verbose=verbose) + copied_depth_map_paths = copy_images_list( + image_paths=polycam_depth_image_filenames, + image_dir=depth_dir, + num_downscales=num_downscales, + crop_border_pixels=crop_border_pixels, + verbose=verbose, + upscale_factor=upscale_factor, + nearest_neighbor=True, + ) CONSOLE.log("[bold green]:tada: Done upscaling depth maps.") return copied_depth_map_paths def copy_images( - data: Path, image_dir: Path, verbose, crop_factor: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0) + data: Path, + image_dir: Path, + verbose: bool = False, + crop_factor: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0), + num_downscales: int = 0, ) -> OrderedDict[Path, Path]: """Copy images from a directory to a new directory. @@ -328,7 +372,11 @@ def copy_images( sys.exit(1) copied_images = copy_images_list( - image_paths=image_paths, image_dir=image_dir, crop_factor=crop_factor, verbose=verbose + image_paths=image_paths, + image_dir=image_dir, + crop_factor=crop_factor, + verbose=verbose, + num_downscales=num_downscales, ) return OrderedDict((original_path, new_path) for original_path, new_path in zip(image_paths, copied_images)) @@ -340,7 +388,8 @@ def downscale_images( nearest_neighbor: bool = False, verbose: bool = False, ) -> str: - """Downscales the images in the directory. Uses FFMPEG. + """(Now deprecated; much faster integrated into copy_images.) + Downscales the images in the directory. Uses FFMPEG. Assumes images are named frame_00001.png, frame_00002.png, etc. diff --git a/nerfstudio/process_data/video_to_nerfstudio_dataset.py b/nerfstudio/process_data/video_to_nerfstudio_dataset.py index 13e4b6bc4..b1eec97ea 100644 --- a/nerfstudio/process_data/video_to_nerfstudio_dataset.py +++ b/nerfstudio/process_data/video_to_nerfstudio_dataset.py @@ -30,9 +30,8 @@ class VideoToNerfstudioDataset(ColmapConverterToNerfstudioDataset): This script does the following: - 1. Converts the video into images. - 2. Scales images to a specified size. - 3. Calculates the camera poses for each image using `COLMAP `_. + 1. Converts the video into images and downscales them. + 2. Calculates the camera poses for each image using `COLMAP `_. """ num_frames_target: int = 300 @@ -53,14 +52,17 @@ class VideoToNerfstudioDataset(ColmapConverterToNerfstudioDataset): self.data, image_dir=temp_image_dir, num_frames_target=self.num_frames_target, + num_downscales=0, crop_factor=(0.0, 0.0, 0.0, 0.0), verbose=self.verbose, ) else: + # If we're not dealing with equirects we can downscale in one step. summary_log, num_extracted_frames = process_data_utils.convert_video_to_images( self.data, image_dir=self.image_dir, num_frames_target=self.num_frames_target, + num_downscales=self.num_downscales, crop_factor=self.crop_factor, verbose=self.verbose, ) @@ -90,6 +92,11 @@ class VideoToNerfstudioDataset(ColmapConverterToNerfstudioDataset): self.camera_type = "perspective" + # # Downscale images + summary_log.append( + process_data_utils.downscale_images(self.image_dir, self.num_downscales, verbose=self.verbose) + ) + # # Create mask mask_path = process_data_utils.save_mask( image_dir=self.image_dir, @@ -100,11 +107,6 @@ class VideoToNerfstudioDataset(ColmapConverterToNerfstudioDataset): if mask_path is not None: summary_log.append(f"Saved mask to {mask_path}") - # # Downscale images - summary_log.append( - process_data_utils.downscale_images(self.image_dir, self.num_downscales, verbose=self.verbose) - ) - # Run Colmap if not self.skip_colmap: self._run_colmap(mask_path) diff --git a/nerfstudio/scripts/process_data.py b/nerfstudio/scripts/process_data.py index 50a10bb5a..201eb94fc 100644 --- a/nerfstudio/scripts/process_data.py +++ b/nerfstudio/scripts/process_data.py @@ -89,7 +89,10 @@ class ProcessRecord3D(BaseConverterToNerfstudioDataset): record3d_image_filenames = list(np.array(record3d_image_filenames)[idx]) # Copy images to output directory copied_image_paths = process_data_utils.copy_images_list( - record3d_image_filenames, image_dir=image_dir, verbose=self.verbose + record3d_image_filenames, + image_dir=image_dir, + verbose=self.verbose, + num_downscales=self.num_downscales, ) num_frames = len(copied_image_paths) @@ -101,9 +104,6 @@ class ProcessRecord3D(BaseConverterToNerfstudioDataset): f"larger than the current value ({self.max_dataset_size}), or -1 to use all images." ) - # Downscale images - summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose)) - metadata_path = self.data / "metadata.json" record3d_utils.record3d_to_json(copied_image_paths, metadata_path, self.output_dir, indices=idx) CONSOLE.rule("[bold green]:tada: :tada: :tada: All DONE :tada: :tada: :tada:") @@ -266,6 +266,7 @@ class ProcessMetashape(BaseConverterToNerfstudioDataset, _NoDefaultProcessMetash image_filenames, image_dir=image_dir, verbose=self.verbose, + num_downscales=self.num_downscales, ) num_frames = len(copied_image_paths) @@ -282,9 +283,6 @@ class ProcessMetashape(BaseConverterToNerfstudioDataset, _NoDefaultProcessMetash else: summary_log.append(f"Started with {num_frames} images") - # Downscale images - summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose)) - # Save json if num_frames == 0: CONSOLE.print("[bold red]No images found, exiting") @@ -353,6 +351,7 @@ class ProcessRealityCapture(BaseConverterToNerfstudioDataset, _NoDefaultProcessR image_filenames, image_dir=image_dir, verbose=self.verbose, + num_downscales=self.num_downscales, ) num_frames = len(copied_image_paths) @@ -369,9 +368,6 @@ class ProcessRealityCapture(BaseConverterToNerfstudioDataset, _NoDefaultProcessR else: summary_log.append(f"Started with {num_frames} images") - # Downscale images - summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose)) - # Save json if num_frames == 0: CONSOLE.print("[bold red]No images found, exiting")