mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-20 06:35:36 +08:00
Fast downscaling for video and image processing (PR2168 plus images) (#2213)
* Merge video image extraction and downscaling * Linting * Fix extracting all frames, only apply vfr when thumnailing * integrate downscale chains into image processing * fast polycam upscale/downscale depth * black * black * Update process_data_utils.py Remove code change from other branch. * use same suffix for downscale, don't force png * black --------- Co-authored-by: Sebastiaan Meijer <seb@stiaan.io>
This commit is contained in:
1 parent
edbe319924
commit
91291bc52c
5 files changed
+129
-79
No files matched your search
@@ -60,16 +60,24 @@ class ImagesToNerfstudioDataset(ColmapConverterToNerfstudioDataset):
|
||||
if not self.skip_image_processing:
|
||||
# Copy images to output directory
|
||||
image_rename_map_paths = process_data_utils.copy_images(
|
||||
self.data, image_dir=self.image_dir, crop_factor=self.crop_factor, verbose=self.verbose
|
||||
self.data,
|
||||
image_dir=self.image_dir,
|
||||
crop_factor=self.crop_factor,
|
||||
verbose=self.verbose,
|
||||
num_downscales=self.num_downscales,
|
||||
)
|
||||
image_rename_map = dict((a.name, b.name) for a, b in image_rename_map_paths.items())
|
||||
num_frames = len(image_rename_map)
|
||||
summary_log.append(f"Starting with {num_frames} images")
|
||||
|
||||
# Downscale images
|
||||
summary_log.append(
|
||||
process_data_utils.downscale_images(self.image_dir, self.num_downscales, verbose=self.verbose)
|
||||
# # Create mask
|
||||
mask_path = process_data_utils.save_mask(
|
||||
image_dir=self.image_dir,
|
||||
num_downscales=self.num_downscales,
|
||||
crop_factor=(0.0, 0.0, 0.0, 0.0),
|
||||
)
|
||||
if mask_path is not None:
|
||||
summary_log.append("Saved mask(s)")
|
||||
else:
|
||||
num_frames = len(process_data_utils.list_images(self.data))
|
||||
if num_frames == 0:
|
||||
|
||||
@@ -131,6 +131,7 @@ def process_images(
|
||||
image_dir=image_dir,
|
||||
crop_border_pixels=crop_border_pixels,
|
||||
verbose=verbose,
|
||||
num_downscales=num_downscales,
|
||||
)
|
||||
num_frames = len(copied_image_paths)
|
||||
|
||||
@@ -145,9 +146,6 @@ def process_images(
|
||||
else:
|
||||
summary_log.append(f"Started with {num_frames} images")
|
||||
|
||||
# Downscale images
|
||||
summary_log.append(process_data_utils.downscale_images(image_dir, num_downscales, verbose=verbose))
|
||||
|
||||
# Save json
|
||||
if num_frames == 0:
|
||||
CONSOLE.print("[bold red]No images found, exiting")
|
||||
@@ -190,7 +188,11 @@ def process_depth_maps(
|
||||
|
||||
# Copy depth images to output directory
|
||||
copied_depth_maps_paths = process_data_utils.copy_and_upscale_polycam_depth_maps_list(
|
||||
polycam_depth_maps_filenames, depth_dir=depth_dir, crop_border_pixels=crop_border_pixels, verbose=verbose
|
||||
polycam_depth_maps_filenames,
|
||||
depth_dir=depth_dir,
|
||||
num_downscales=num_downscales,
|
||||
crop_border_pixels=crop_border_pixels,
|
||||
verbose=verbose,
|
||||
)
|
||||
|
||||
num_processed_depth_maps = len(copied_depth_maps_paths)
|
||||
@@ -211,11 +213,4 @@ def process_depth_maps(
|
||||
else:
|
||||
summary_log.append(f"Started with {num_processed_depth_maps} images")
|
||||
|
||||
# Downscale depth maps
|
||||
summary_log.append(
|
||||
process_data_utils.downscale_images(
|
||||
depth_dir, num_downscales, folder_name="depths", nearest_neighbor=True, verbose=verbose
|
||||
)
|
||||
)
|
||||
|
||||
return summary_log, polycam_depth_maps_filenames
|
||||
@@ -111,6 +111,7 @@ def convert_video_to_images(
|
||||
video_path: Path,
|
||||
image_dir: Path,
|
||||
num_frames_target: int,
|
||||
num_downscales: int,
|
||||
crop_factor: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0),
|
||||
verbose: bool = False,
|
||||
) -> Tuple[List[str], int]:
|
||||
@@ -120,6 +121,7 @@ def convert_video_to_images(
|
||||
video_path: Path to the video.
|
||||
output_dir: Path to the output directory.
|
||||
num_frames_target: Number of frames to extract.
|
||||
num_downscales: Number of times to downscale the images. Downscales by 2 each time.
|
||||
crop_factor: Portion of the image to crop. Should be in [0,1] (top, bottom, left, right)
|
||||
verbose: If True, logs the output of the command.
|
||||
Returns:
|
||||
@@ -151,7 +153,6 @@ def convert_video_to_images(
|
||||
sys.exit(1)
|
||||
CONSOLE.print("Number of frames in video:", num_frames)
|
||||
|
||||
out_filename = image_dir / "frame_%05d.png"
|
||||
ffmpeg_cmd = f'ffmpeg -i "{video_path}"'
|
||||
|
||||
crop_cmd = ""
|
||||
@@ -160,41 +161,66 @@ def convert_video_to_images(
|
||||
width = 1 - crop_factor[2] - crop_factor[3]
|
||||
start_x = crop_factor[2]
|
||||
start_y = crop_factor[0]
|
||||
crop_cmd = f',"crop=w=iw*{width}:h=ih*{height}:x=iw*{start_x}:y=ih*{start_y}"'
|
||||
crop_cmd = f"crop=w=iw*{width}:h=ih*{height}:x=iw*{start_x}:y=ih*{start_y},"
|
||||
|
||||
num_frames = get_num_frames_in_video(video_path)
|
||||
spacing = num_frames // num_frames_target
|
||||
|
||||
downscale_chains = [f"[t{i}]scale=iw/{2**i}:ih/{2**i}[out{i}]" for i in range(num_downscales + 1)]
|
||||
downscale_dirs = [Path(str(image_dir) + (f"_{2**i}" if i > 0 else "")) for i in range(num_downscales + 1)]
|
||||
downscale_paths = [downscale_dirs[i] / "frame_%05d.png" for i in range(num_downscales + 1)]
|
||||
|
||||
for dir in downscale_dirs:
|
||||
dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
downscale_chain = (
|
||||
f"split={num_downscales + 1}"
|
||||
+ "".join([f"[t{i}]" for i in range(num_downscales + 1)])
|
||||
+ ";"
|
||||
+ ";".join(downscale_chains)
|
||||
)
|
||||
|
||||
if spacing > 1:
|
||||
ffmpeg_cmd += f" -vf thumbnail={spacing},setpts=N/TB{crop_cmd} -r 1"
|
||||
CONSOLE.print("Number of frames to extract:", math.ceil(num_frames / spacing))
|
||||
ffmpeg_cmd += " -vsync vfr -r1"
|
||||
select_cmd = f"thumbnail={spacing},setpts=N/TB,"
|
||||
else:
|
||||
CONSOLE.print("[bold red]Can't satisfy requested number of frames. Extracting all frames.")
|
||||
ffmpeg_cmd += " -pix_fmt bgr8"
|
||||
if crop_cmd != "":
|
||||
ffmpeg_cmd += f" -vf {crop_cmd[1:]}"
|
||||
select_cmd = ""
|
||||
|
||||
downscale_cmd = f' -filter_complex "{select_cmd}{crop_cmd}{downscale_chain}"' + "".join(
|
||||
[f' -map "[out{i}]" "{downscale_paths[i]}"' for i in range(num_downscales + 1)]
|
||||
)
|
||||
|
||||
ffmpeg_cmd += downscale_cmd
|
||||
|
||||
ffmpeg_cmd += f" {out_filename}"
|
||||
run_command(ffmpeg_cmd, verbose=verbose)
|
||||
|
||||
num_final_frames = len(list(image_dir.glob("*.png")))
|
||||
summary_log = []
|
||||
summary_log.append(f"Starting with {num_frames} video frames")
|
||||
summary_log.append(f"We extracted {num_final_frames} images")
|
||||
CONSOLE.log("[bold green]:tada: Done converting video to images.")
|
||||
num_final_frames = len(list(image_dir.glob("*.png")))
|
||||
summary_log = []
|
||||
summary_log.append(f"Starting with {num_frames} video frames")
|
||||
summary_log.append(f"We extracted {num_final_frames} images")
|
||||
CONSOLE.log("[bold green]:tada: Done converting video to images.")
|
||||
|
||||
return summary_log, num_final_frames
|
||||
return summary_log, num_final_frames
|
||||
|
||||
|
||||
def copy_images_list(
|
||||
image_paths: List[Path],
|
||||
image_dir: Path,
|
||||
num_downscales: int,
|
||||
crop_border_pixels: Optional[int] = None,
|
||||
crop_factor: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0),
|
||||
verbose: bool = False,
|
||||
upscale_factor: Optional[int] = None,
|
||||
nearest_neighbor: bool = False,
|
||||
) -> List[Path]:
|
||||
"""Copy all images in a list of Paths. Useful for filtering from a directory.
|
||||
Args:
|
||||
image_paths: List of Paths of images to copy to a new directory.
|
||||
image_dir: Path to the output directory.
|
||||
num_downscales: Number of times to downscale the images. Downscales by 2 each time.
|
||||
crop_border_pixels: If not None, crops each edge by the specified number of pixels.
|
||||
crop_factor: Portion of the image to crop. Should be in [0,1] (top, bottom, left, right)
|
||||
verbose: If True, print extra logging.
|
||||
@@ -230,26 +256,48 @@ def copy_images_list(
|
||||
pass
|
||||
copied_image_paths.append(copied_image_path)
|
||||
|
||||
nn_flag = "" if not nearest_neighbor else ":flags=neighbor"
|
||||
downscale_chains = [f"[t{i}]scale=iw/{2**i}:ih/{2**i}{nn_flag}[out{i}]" for i in range(num_downscales + 1)]
|
||||
downscale_dirs = [Path(str(image_dir) + (f"_{2**i}" if i > 0 else "")) for i in range(num_downscales + 1)]
|
||||
downscale_paths = [
|
||||
downscale_dirs[i] / ("frame_%05d" + copied_image_paths[0].suffix) for i in range(num_downscales + 1)
|
||||
]
|
||||
|
||||
for dir in downscale_dirs:
|
||||
dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
downscale_chain = (
|
||||
f"split={num_downscales + 1}"
|
||||
+ "".join([f"[t{i}]" for i in range(num_downscales + 1)])
|
||||
+ ";"
|
||||
+ ";".join(downscale_chains)
|
||||
)
|
||||
|
||||
num_frames = len(image_paths)
|
||||
ffmpeg_cmd = f'ffmpeg -y -noautorotate -i "{image_dir / f"frame_%05d{copied_image_paths[0].suffix}"}" -q:v 2 '
|
||||
|
||||
crop_cmd = ""
|
||||
if crop_border_pixels is not None:
|
||||
file_type = image_paths[0].suffix
|
||||
filename = f"frame_%05d{file_type}"
|
||||
crop = f"crop=iw-{crop_border_pixels*2}:ih-{crop_border_pixels*2}"
|
||||
ffmpeg_cmd = f'ffmpeg -y -noautorotate -i "{image_dir / filename}" -q:v 2 -vf {crop} "{image_dir / filename}"'
|
||||
run_command(ffmpeg_cmd, verbose=verbose)
|
||||
crop_cmd = f"crop=iw-{crop_border_pixels*2}:ih-{crop_border_pixels*2}[cropped];[cropped]"
|
||||
elif crop_factor != (0.0, 0.0, 0.0, 0.0):
|
||||
file_type = image_paths[0].suffix
|
||||
filename = f"frame_%05d{file_type}"
|
||||
height = 1 - crop_factor[0] - crop_factor[1]
|
||||
width = 1 - crop_factor[2] - crop_factor[3]
|
||||
start_x = crop_factor[2]
|
||||
start_y = crop_factor[0]
|
||||
crop_cmd = f',"crop=w=iw*{width}:h=ih*{height}:x=iw*{start_x}:y=ih*{start_y}"'
|
||||
ffmpeg_cmd = (
|
||||
f'ffmpeg -y -noautorotate -i "{image_dir / filename}" -q:v 2 -vf {crop_cmd[1:]} "{image_dir / filename}"'
|
||||
)
|
||||
run_command(ffmpeg_cmd, verbose=verbose)
|
||||
crop_cmd = f"crop=w=iw*{width}:h=ih*{height}:x=iw*{start_x}:y=ih*{start_y}[cropped];[cropped]"
|
||||
|
||||
num_frames = len(image_paths)
|
||||
select_cmd = "[0:v]"
|
||||
if upscale_factor is not None:
|
||||
select_cmd = f"[0:v]scale=iw*{upscale_factor}:ih*{upscale_factor}:flags=neighbor[upscaled];[upscaled]"
|
||||
|
||||
downscale_cmd = f' -filter_complex "{select_cmd}{crop_cmd}{downscale_chain}"' + "".join(
|
||||
[f' -map "[out{i}]" "{downscale_paths[i]}"' for i in range(num_downscales + 1)]
|
||||
)
|
||||
|
||||
ffmpeg_cmd += downscale_cmd
|
||||
if verbose:
|
||||
CONSOLE.log(f"... {ffmpeg_cmd}")
|
||||
run_command(ffmpeg_cmd, verbose=verbose)
|
||||
|
||||
if num_frames == 0:
|
||||
CONSOLE.log("[bold red]:skull: No usable images in the data folder.")
|
||||
@@ -262,6 +310,7 @@ def copy_images_list(
|
||||
def copy_and_upscale_polycam_depth_maps_list(
|
||||
polycam_depth_image_filenames: List[Path],
|
||||
depth_dir: Path,
|
||||
num_downscales: int,
|
||||
crop_border_pixels: Optional[int] = None,
|
||||
verbose: bool = False,
|
||||
) -> List[Path]:
|
||||
@@ -284,31 +333,26 @@ def copy_and_upscale_polycam_depth_maps_list(
|
||||
assert upscale_factor > 1
|
||||
assert isinstance(upscale_factor, int)
|
||||
|
||||
copied_depth_map_paths = []
|
||||
for idx, depth_map in enumerate(polycam_depth_image_filenames):
|
||||
destination = depth_dir / f"frame_{idx + 1:05d}{depth_map.suffix}"
|
||||
ffmpeg_cmd = [
|
||||
f'ffmpeg -y -i "{depth_map}" ',
|
||||
f"-q:v 2 -vf scale=iw*{upscale_factor}:ih*{upscale_factor}:flags=neighbor ",
|
||||
f'"{destination}"',
|
||||
]
|
||||
ffmpeg_cmd = " ".join(ffmpeg_cmd)
|
||||
run_command(ffmpeg_cmd, verbose=verbose)
|
||||
copied_depth_map_paths.append(destination)
|
||||
|
||||
if crop_border_pixels is not None:
|
||||
file_type = depth_dir.glob("frame_*").__next__().suffix
|
||||
filename = f"frame_%05d{file_type}"
|
||||
crop = f"crop=iw-{crop_border_pixels * 2}:ih-{crop_border_pixels * 2}"
|
||||
ffmpeg_cmd = f'ffmpeg -y -i "{depth_dir / filename}" -q:v 2 -vf {crop} "{depth_dir / filename}"'
|
||||
run_command(ffmpeg_cmd, verbose=verbose)
|
||||
copied_depth_map_paths = copy_images_list(
|
||||
image_paths=polycam_depth_image_filenames,
|
||||
image_dir=depth_dir,
|
||||
num_downscales=num_downscales,
|
||||
crop_border_pixels=crop_border_pixels,
|
||||
verbose=verbose,
|
||||
upscale_factor=upscale_factor,
|
||||
nearest_neighbor=True,
|
||||
)
|
||||
|
||||
CONSOLE.log("[bold green]:tada: Done upscaling depth maps.")
|
||||
return copied_depth_map_paths
|
||||
|
||||
|
||||
def copy_images(
|
||||
data: Path, image_dir: Path, verbose, crop_factor: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0)
|
||||
data: Path,
|
||||
image_dir: Path,
|
||||
verbose: bool = False,
|
||||
crop_factor: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0),
|
||||
num_downscales: int = 0,
|
||||
) -> OrderedDict[Path, Path]:
|
||||
"""Copy images from a directory to a new directory.
|
||||
|
||||
@@ -328,7 +372,11 @@ def copy_images(
|
||||
sys.exit(1)
|
||||
|
||||
copied_images = copy_images_list(
|
||||
image_paths=image_paths, image_dir=image_dir, crop_factor=crop_factor, verbose=verbose
|
||||
image_paths=image_paths,
|
||||
image_dir=image_dir,
|
||||
crop_factor=crop_factor,
|
||||
verbose=verbose,
|
||||
num_downscales=num_downscales,
|
||||
)
|
||||
return OrderedDict((original_path, new_path) for original_path, new_path in zip(image_paths, copied_images))
|
||||
|
||||
@@ -340,7 +388,8 @@ def downscale_images(
|
||||
nearest_neighbor: bool = False,
|
||||
verbose: bool = False,
|
||||
) -> str:
|
||||
"""Downscales the images in the directory. Uses FFMPEG.
|
||||
"""(Now deprecated; much faster integrated into copy_images.)
|
||||
Downscales the images in the directory. Uses FFMPEG.
|
||||
|
||||
Assumes images are named frame_00001.png, frame_00002.png, etc.
|
||||
|
||||
|
||||
@@ -30,9 +30,8 @@ class VideoToNerfstudioDataset(ColmapConverterToNerfstudioDataset):
|
||||
|
||||
This script does the following:
|
||||
|
||||
1. Converts the video into images.
|
||||
2. Scales images to a specified size.
|
||||
3. Calculates the camera poses for each image using `COLMAP <https://colmap.github.io/>`_.
|
||||
1. Converts the video into images and downscales them.
|
||||
2. Calculates the camera poses for each image using `COLMAP <https://colmap.github.io/>`_.
|
||||
"""
|
||||
|
||||
num_frames_target: int = 300
|
||||
@@ -53,14 +52,17 @@ class VideoToNerfstudioDataset(ColmapConverterToNerfstudioDataset):
|
||||
self.data,
|
||||
image_dir=temp_image_dir,
|
||||
num_frames_target=self.num_frames_target,
|
||||
num_downscales=0,
|
||||
crop_factor=(0.0, 0.0, 0.0, 0.0),
|
||||
verbose=self.verbose,
|
||||
)
|
||||
else:
|
||||
# If we're not dealing with equirects we can downscale in one step.
|
||||
summary_log, num_extracted_frames = process_data_utils.convert_video_to_images(
|
||||
self.data,
|
||||
image_dir=self.image_dir,
|
||||
num_frames_target=self.num_frames_target,
|
||||
num_downscales=self.num_downscales,
|
||||
crop_factor=self.crop_factor,
|
||||
verbose=self.verbose,
|
||||
)
|
||||
@@ -90,6 +92,11 @@ class VideoToNerfstudioDataset(ColmapConverterToNerfstudioDataset):
|
||||
|
||||
self.camera_type = "perspective"
|
||||
|
||||
# # Downscale images
|
||||
summary_log.append(
|
||||
process_data_utils.downscale_images(self.image_dir, self.num_downscales, verbose=self.verbose)
|
||||
)
|
||||
|
||||
# # Create mask
|
||||
mask_path = process_data_utils.save_mask(
|
||||
image_dir=self.image_dir,
|
||||
@@ -100,11 +107,6 @@ class VideoToNerfstudioDataset(ColmapConverterToNerfstudioDataset):
|
||||
if mask_path is not None:
|
||||
summary_log.append(f"Saved mask to {mask_path}")
|
||||
|
||||
# # Downscale images
|
||||
summary_log.append(
|
||||
process_data_utils.downscale_images(self.image_dir, self.num_downscales, verbose=self.verbose)
|
||||
)
|
||||
|
||||
# Run Colmap
|
||||
if not self.skip_colmap:
|
||||
self._run_colmap(mask_path)
|
||||
|
||||
@@ -89,7 +89,10 @@ class ProcessRecord3D(BaseConverterToNerfstudioDataset):
|
||||
record3d_image_filenames = list(np.array(record3d_image_filenames)[idx])
|
||||
# Copy images to output directory
|
||||
copied_image_paths = process_data_utils.copy_images_list(
|
||||
record3d_image_filenames, image_dir=image_dir, verbose=self.verbose
|
||||
record3d_image_filenames,
|
||||
image_dir=image_dir,
|
||||
verbose=self.verbose,
|
||||
num_downscales=self.num_downscales,
|
||||
)
|
||||
num_frames = len(copied_image_paths)
|
||||
|
||||
@@ -101,9 +104,6 @@ class ProcessRecord3D(BaseConverterToNerfstudioDataset):
|
||||
f"larger than the current value ({self.max_dataset_size}), or -1 to use all images."
|
||||
)
|
||||
|
||||
# Downscale images
|
||||
summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose))
|
||||
|
||||
metadata_path = self.data / "metadata.json"
|
||||
record3d_utils.record3d_to_json(copied_image_paths, metadata_path, self.output_dir, indices=idx)
|
||||
CONSOLE.rule("[bold green]:tada: :tada: :tada: All DONE :tada: :tada: :tada:")
|
||||
@@ -266,6 +266,7 @@ class ProcessMetashape(BaseConverterToNerfstudioDataset, _NoDefaultProcessMetash
|
||||
image_filenames,
|
||||
image_dir=image_dir,
|
||||
verbose=self.verbose,
|
||||
num_downscales=self.num_downscales,
|
||||
)
|
||||
num_frames = len(copied_image_paths)
|
||||
|
||||
@@ -282,9 +283,6 @@ class ProcessMetashape(BaseConverterToNerfstudioDataset, _NoDefaultProcessMetash
|
||||
else:
|
||||
summary_log.append(f"Started with {num_frames} images")
|
||||
|
||||
# Downscale images
|
||||
summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose))
|
||||
|
||||
# Save json
|
||||
if num_frames == 0:
|
||||
CONSOLE.print("[bold red]No images found, exiting")
|
||||
@@ -353,6 +351,7 @@ class ProcessRealityCapture(BaseConverterToNerfstudioDataset, _NoDefaultProcessR
|
||||
image_filenames,
|
||||
image_dir=image_dir,
|
||||
verbose=self.verbose,
|
||||
num_downscales=self.num_downscales,
|
||||
)
|
||||
num_frames = len(copied_image_paths)
|
||||
|
||||
@@ -369,9 +368,6 @@ class ProcessRealityCapture(BaseConverterToNerfstudioDataset, _NoDefaultProcessR
|
||||
else:
|
||||
summary_log.append(f"Started with {num_frames} images")
|
||||
|
||||
# Downscale images
|
||||
summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose))
|
||||
|
||||
# Save json
|
||||
if num_frames == 0:
|
||||
CONSOLE.print("[bold red]No images found, exiting")
|
||||
|
||||
Reference in new issue
Block a user