Fast downscaling for video and image processing (PR2168 plus images) (#2213)

* Merge video image extraction and downscaling

* Linting

* Fix extracting all frames, only apply vfr when thumnailing

* integrate downscale chains into image processing

* fast polycam upscale/downscale depth

* black

* black

* Update process_data_utils.py

Remove code change from other branch.

* use same suffix for downscale, don't force png

* black

---------

Co-authored-by: Sebastiaan Meijer <seb@stiaan.io>
This commit is contained in:
machenmusikandSebastiaan Meijer authored and GitHub committed 2023-07-13 18:54:19 -04:00
1 parent edbe319924
commit 91291bc52c
5 files changed
+129 -79

No files matched your search

@@ -60,16 +60,24 @@ class ImagesToNerfstudioDataset(ColmapConverterToNerfstudioDataset):
if not self.skip_image_processing:
# Copy images to output directory
image_rename_map_paths = process_data_utils.copy_images(
self.data, image_dir=self.image_dir, crop_factor=self.crop_factor, verbose=self.verbose
self.data,
image_dir=self.image_dir,
crop_factor=self.crop_factor,
verbose=self.verbose,
num_downscales=self.num_downscales,
)
image_rename_map = dict((a.name, b.name) for a, b in image_rename_map_paths.items())
num_frames = len(image_rename_map)
summary_log.append(f"Starting with {num_frames} images")
# Downscale images
summary_log.append(
process_data_utils.downscale_images(self.image_dir, self.num_downscales, verbose=self.verbose)
# # Create mask
mask_path = process_data_utils.save_mask(
image_dir=self.image_dir,
num_downscales=self.num_downscales,
crop_factor=(0.0, 0.0, 0.0, 0.0),
)
if mask_path is not None:
summary_log.append("Saved mask(s)")
else:
num_frames = len(process_data_utils.list_images(self.data))
if num_frames == 0:
+6 -11
View File
@@ -131,6 +131,7 @@ def process_images(
image_dir=image_dir,
crop_border_pixels=crop_border_pixels,
verbose=verbose,
num_downscales=num_downscales,
)
num_frames = len(copied_image_paths)
@@ -145,9 +146,6 @@ def process_images(
else:
summary_log.append(f"Started with {num_frames} images")
# Downscale images
summary_log.append(process_data_utils.downscale_images(image_dir, num_downscales, verbose=verbose))
# Save json
if num_frames == 0:
CONSOLE.print("[bold red]No images found, exiting")
@@ -190,7 +188,11 @@ def process_depth_maps(
# Copy depth images to output directory
copied_depth_maps_paths = process_data_utils.copy_and_upscale_polycam_depth_maps_list(
polycam_depth_maps_filenames, depth_dir=depth_dir, crop_border_pixels=crop_border_pixels, verbose=verbose
polycam_depth_maps_filenames,
depth_dir=depth_dir,
num_downscales=num_downscales,
crop_border_pixels=crop_border_pixels,
verbose=verbose,
)
num_processed_depth_maps = len(copied_depth_maps_paths)
@@ -211,11 +213,4 @@ def process_depth_maps(
else:
summary_log.append(f"Started with {num_processed_depth_maps} images")
# Downscale depth maps
summary_log.append(
process_data_utils.downscale_images(
depth_dir, num_downscales, folder_name="depths", nearest_neighbor=True, verbose=verbose
)
)
return summary_log, polycam_depth_maps_filenames
+95 -46
View File
@@ -111,6 +111,7 @@ def convert_video_to_images(
video_path: Path,
image_dir: Path,
num_frames_target: int,
num_downscales: int,
crop_factor: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0),
verbose: bool = False,
) -> Tuple[List[str], int]:
@@ -120,6 +121,7 @@ def convert_video_to_images(
video_path: Path to the video.
output_dir: Path to the output directory.
num_frames_target: Number of frames to extract.
num_downscales: Number of times to downscale the images. Downscales by 2 each time.
crop_factor: Portion of the image to crop. Should be in [0,1] (top, bottom, left, right)
verbose: If True, logs the output of the command.
Returns:
@@ -151,7 +153,6 @@ def convert_video_to_images(
sys.exit(1)
CONSOLE.print("Number of frames in video:", num_frames)
out_filename = image_dir / "frame_%05d.png"
ffmpeg_cmd = f'ffmpeg -i "{video_path}"'
crop_cmd = ""
@@ -160,41 +161,66 @@ def convert_video_to_images(
width = 1 - crop_factor[2] - crop_factor[3]
start_x = crop_factor[2]
start_y = crop_factor[0]
crop_cmd = f',"crop=w=iw*{width}:h=ih*{height}:x=iw*{start_x}:y=ih*{start_y}"'
crop_cmd = f"crop=w=iw*{width}:h=ih*{height}:x=iw*{start_x}:y=ih*{start_y},"
num_frames = get_num_frames_in_video(video_path)
spacing = num_frames // num_frames_target
downscale_chains = [f"[t{i}]scale=iw/{2**i}:ih/{2**i}[out{i}]" for i in range(num_downscales + 1)]
downscale_dirs = [Path(str(image_dir) + (f"_{2**i}" if i > 0 else "")) for i in range(num_downscales + 1)]
downscale_paths = [downscale_dirs[i] / "frame_%05d.png" for i in range(num_downscales + 1)]
for dir in downscale_dirs:
dir.mkdir(parents=True, exist_ok=True)
downscale_chain = (
f"split={num_downscales + 1}"
+ "".join([f"[t{i}]" for i in range(num_downscales + 1)])
+ ";"
+ ";".join(downscale_chains)
)
if spacing > 1:
ffmpeg_cmd += f" -vf thumbnail={spacing},setpts=N/TB{crop_cmd} -r 1"
CONSOLE.print("Number of frames to extract:", math.ceil(num_frames / spacing))
ffmpeg_cmd += " -vsync vfr -r1"
select_cmd = f"thumbnail={spacing},setpts=N/TB,"
else:
CONSOLE.print("[bold red]Can't satisfy requested number of frames. Extracting all frames.")
ffmpeg_cmd += " -pix_fmt bgr8"
if crop_cmd != "":
ffmpeg_cmd += f" -vf {crop_cmd[1:]}"
select_cmd = ""
downscale_cmd = f' -filter_complex "{select_cmd}{crop_cmd}{downscale_chain}"' + "".join(
[f' -map "[out{i}]" "{downscale_paths[i]}"' for i in range(num_downscales + 1)]
)
ffmpeg_cmd += downscale_cmd
ffmpeg_cmd += f" {out_filename}"
run_command(ffmpeg_cmd, verbose=verbose)
num_final_frames = len(list(image_dir.glob("*.png")))
summary_log = []
summary_log.append(f"Starting with {num_frames} video frames")
summary_log.append(f"We extracted {num_final_frames} images")
CONSOLE.log("[bold green]:tada: Done converting video to images.")
num_final_frames = len(list(image_dir.glob("*.png")))
summary_log = []
summary_log.append(f"Starting with {num_frames} video frames")
summary_log.append(f"We extracted {num_final_frames} images")
CONSOLE.log("[bold green]:tada: Done converting video to images.")
return summary_log, num_final_frames
return summary_log, num_final_frames
def copy_images_list(
image_paths: List[Path],
image_dir: Path,
num_downscales: int,
crop_border_pixels: Optional[int] = None,
crop_factor: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0),
verbose: bool = False,
upscale_factor: Optional[int] = None,
nearest_neighbor: bool = False,
) -> List[Path]:
"""Copy all images in a list of Paths. Useful for filtering from a directory.
Args:
image_paths: List of Paths of images to copy to a new directory.
image_dir: Path to the output directory.
num_downscales: Number of times to downscale the images. Downscales by 2 each time.
crop_border_pixels: If not None, crops each edge by the specified number of pixels.
crop_factor: Portion of the image to crop. Should be in [0,1] (top, bottom, left, right)
verbose: If True, print extra logging.
@@ -230,26 +256,48 @@ def copy_images_list(
pass
copied_image_paths.append(copied_image_path)
nn_flag = "" if not nearest_neighbor else ":flags=neighbor"
downscale_chains = [f"[t{i}]scale=iw/{2**i}:ih/{2**i}{nn_flag}[out{i}]" for i in range(num_downscales + 1)]
downscale_dirs = [Path(str(image_dir) + (f"_{2**i}" if i > 0 else "")) for i in range(num_downscales + 1)]
downscale_paths = [
downscale_dirs[i] / ("frame_%05d" + copied_image_paths[0].suffix) for i in range(num_downscales + 1)
]
for dir in downscale_dirs:
dir.mkdir(parents=True, exist_ok=True)
downscale_chain = (
f"split={num_downscales + 1}"
+ "".join([f"[t{i}]" for i in range(num_downscales + 1)])
+ ";"
+ ";".join(downscale_chains)
)
num_frames = len(image_paths)
ffmpeg_cmd = f'ffmpeg -y -noautorotate -i "{image_dir / f"frame_%05d{copied_image_paths[0].suffix}"}" -q:v 2 '
crop_cmd = ""
if crop_border_pixels is not None:
file_type = image_paths[0].suffix
filename = f"frame_%05d{file_type}"
crop = f"crop=iw-{crop_border_pixels*2}:ih-{crop_border_pixels*2}"
ffmpeg_cmd = f'ffmpeg -y -noautorotate -i "{image_dir / filename}" -q:v 2 -vf {crop} "{image_dir / filename}"'
run_command(ffmpeg_cmd, verbose=verbose)
crop_cmd = f"crop=iw-{crop_border_pixels*2}:ih-{crop_border_pixels*2}[cropped];[cropped]"
elif crop_factor != (0.0, 0.0, 0.0, 0.0):
file_type = image_paths[0].suffix
filename = f"frame_%05d{file_type}"
height = 1 - crop_factor[0] - crop_factor[1]
width = 1 - crop_factor[2] - crop_factor[3]
start_x = crop_factor[2]
start_y = crop_factor[0]
crop_cmd = f',"crop=w=iw*{width}:h=ih*{height}:x=iw*{start_x}:y=ih*{start_y}"'
ffmpeg_cmd = (
f'ffmpeg -y -noautorotate -i "{image_dir / filename}" -q:v 2 -vf {crop_cmd[1:]} "{image_dir / filename}"'
)
run_command(ffmpeg_cmd, verbose=verbose)
crop_cmd = f"crop=w=iw*{width}:h=ih*{height}:x=iw*{start_x}:y=ih*{start_y}[cropped];[cropped]"
num_frames = len(image_paths)
select_cmd = "[0:v]"
if upscale_factor is not None:
select_cmd = f"[0:v]scale=iw*{upscale_factor}:ih*{upscale_factor}:flags=neighbor[upscaled];[upscaled]"
downscale_cmd = f' -filter_complex "{select_cmd}{crop_cmd}{downscale_chain}"' + "".join(
[f' -map "[out{i}]" "{downscale_paths[i]}"' for i in range(num_downscales + 1)]
)
ffmpeg_cmd += downscale_cmd
if verbose:
CONSOLE.log(f"... {ffmpeg_cmd}")
run_command(ffmpeg_cmd, verbose=verbose)
if num_frames == 0:
CONSOLE.log("[bold red]:skull: No usable images in the data folder.")
@@ -262,6 +310,7 @@ def copy_images_list(
def copy_and_upscale_polycam_depth_maps_list(
polycam_depth_image_filenames: List[Path],
depth_dir: Path,
num_downscales: int,
crop_border_pixels: Optional[int] = None,
verbose: bool = False,
) -> List[Path]:
@@ -284,31 +333,26 @@ def copy_and_upscale_polycam_depth_maps_list(
assert upscale_factor > 1
assert isinstance(upscale_factor, int)
copied_depth_map_paths = []
for idx, depth_map in enumerate(polycam_depth_image_filenames):
destination = depth_dir / f"frame_{idx + 1:05d}{depth_map.suffix}"
ffmpeg_cmd = [
f'ffmpeg -y -i "{depth_map}" ',
f"-q:v 2 -vf scale=iw*{upscale_factor}:ih*{upscale_factor}:flags=neighbor ",
f'"{destination}"',
]
ffmpeg_cmd = " ".join(ffmpeg_cmd)
run_command(ffmpeg_cmd, verbose=verbose)
copied_depth_map_paths.append(destination)
if crop_border_pixels is not None:
file_type = depth_dir.glob("frame_*").__next__().suffix
filename = f"frame_%05d{file_type}"
crop = f"crop=iw-{crop_border_pixels * 2}:ih-{crop_border_pixels * 2}"
ffmpeg_cmd = f'ffmpeg -y -i "{depth_dir / filename}" -q:v 2 -vf {crop} "{depth_dir / filename}"'
run_command(ffmpeg_cmd, verbose=verbose)
copied_depth_map_paths = copy_images_list(
image_paths=polycam_depth_image_filenames,
image_dir=depth_dir,
num_downscales=num_downscales,
crop_border_pixels=crop_border_pixels,
verbose=verbose,
upscale_factor=upscale_factor,
nearest_neighbor=True,
)
CONSOLE.log("[bold green]:tada: Done upscaling depth maps.")
return copied_depth_map_paths
def copy_images(
data: Path, image_dir: Path, verbose, crop_factor: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0)
data: Path,
image_dir: Path,
verbose: bool = False,
crop_factor: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0),
num_downscales: int = 0,
) -> OrderedDict[Path, Path]:
"""Copy images from a directory to a new directory.
@@ -328,7 +372,11 @@ def copy_images(
sys.exit(1)
copied_images = copy_images_list(
image_paths=image_paths, image_dir=image_dir, crop_factor=crop_factor, verbose=verbose
image_paths=image_paths,
image_dir=image_dir,
crop_factor=crop_factor,
verbose=verbose,
num_downscales=num_downscales,
)
return OrderedDict((original_path, new_path) for original_path, new_path in zip(image_paths, copied_images))
@@ -340,7 +388,8 @@ def downscale_images(
nearest_neighbor: bool = False,
verbose: bool = False,
) -> str:
"""Downscales the images in the directory. Uses FFMPEG.
"""(Now deprecated; much faster integrated into copy_images.)
Downscales the images in the directory. Uses FFMPEG.
Assumes images are named frame_00001.png, frame_00002.png, etc.
@@ -30,9 +30,8 @@ class VideoToNerfstudioDataset(ColmapConverterToNerfstudioDataset):
This script does the following:
1. Converts the video into images.
2. Scales images to a specified size.
3. Calculates the camera poses for each image using `COLMAP <https://colmap.github.io/>`_.
1. Converts the video into images and downscales them.
2. Calculates the camera poses for each image using `COLMAP <https://colmap.github.io/>`_.
"""
num_frames_target: int = 300
@@ -53,14 +52,17 @@ class VideoToNerfstudioDataset(ColmapConverterToNerfstudioDataset):
self.data,
image_dir=temp_image_dir,
num_frames_target=self.num_frames_target,
num_downscales=0,
crop_factor=(0.0, 0.0, 0.0, 0.0),
verbose=self.verbose,
)
else:
# If we're not dealing with equirects we can downscale in one step.
summary_log, num_extracted_frames = process_data_utils.convert_video_to_images(
self.data,
image_dir=self.image_dir,
num_frames_target=self.num_frames_target,
num_downscales=self.num_downscales,
crop_factor=self.crop_factor,
verbose=self.verbose,
)
@@ -90,6 +92,11 @@ class VideoToNerfstudioDataset(ColmapConverterToNerfstudioDataset):
self.camera_type = "perspective"
# # Downscale images
summary_log.append(
process_data_utils.downscale_images(self.image_dir, self.num_downscales, verbose=self.verbose)
)
# # Create mask
mask_path = process_data_utils.save_mask(
image_dir=self.image_dir,
@@ -100,11 +107,6 @@ class VideoToNerfstudioDataset(ColmapConverterToNerfstudioDataset):
if mask_path is not None:
summary_log.append(f"Saved mask to {mask_path}")
# # Downscale images
summary_log.append(
process_data_utils.downscale_images(self.image_dir, self.num_downscales, verbose=self.verbose)
)
# Run Colmap
if not self.skip_colmap:
self._run_colmap(mask_path)
+6 -10
View File
@@ -89,7 +89,10 @@ class ProcessRecord3D(BaseConverterToNerfstudioDataset):
record3d_image_filenames = list(np.array(record3d_image_filenames)[idx])
# Copy images to output directory
copied_image_paths = process_data_utils.copy_images_list(
record3d_image_filenames, image_dir=image_dir, verbose=self.verbose
record3d_image_filenames,
image_dir=image_dir,
verbose=self.verbose,
num_downscales=self.num_downscales,
)
num_frames = len(copied_image_paths)
@@ -101,9 +104,6 @@ class ProcessRecord3D(BaseConverterToNerfstudioDataset):
f"larger than the current value ({self.max_dataset_size}), or -1 to use all images."
)
# Downscale images
summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose))
metadata_path = self.data / "metadata.json"
record3d_utils.record3d_to_json(copied_image_paths, metadata_path, self.output_dir, indices=idx)
CONSOLE.rule("[bold green]:tada: :tada: :tada: All DONE :tada: :tada: :tada:")
@@ -266,6 +266,7 @@ class ProcessMetashape(BaseConverterToNerfstudioDataset, _NoDefaultProcessMetash
image_filenames,
image_dir=image_dir,
verbose=self.verbose,
num_downscales=self.num_downscales,
)
num_frames = len(copied_image_paths)
@@ -282,9 +283,6 @@ class ProcessMetashape(BaseConverterToNerfstudioDataset, _NoDefaultProcessMetash
else:
summary_log.append(f"Started with {num_frames} images")
# Downscale images
summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose))
# Save json
if num_frames == 0:
CONSOLE.print("[bold red]No images found, exiting")
@@ -353,6 +351,7 @@ class ProcessRealityCapture(BaseConverterToNerfstudioDataset, _NoDefaultProcessR
image_filenames,
image_dir=image_dir,
verbose=self.verbose,
num_downscales=self.num_downscales,
)
num_frames = len(copied_image_paths)
@@ -369,9 +368,6 @@ class ProcessRealityCapture(BaseConverterToNerfstudioDataset, _NoDefaultProcessR
else:
summary_log.append(f"Started with {num_frames} images")
# Downscale images
summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose))
# Save json
if num_frames == 0:
CONSOLE.print("[bold red]No images found, exiting")