diff --git a/nerfstudio/process_data/images_to_nerfstudio_dataset.py b/nerfstudio/process_data/images_to_nerfstudio_dataset.py
index 4c0a5bf03..dc613412a 100644
--- a/nerfstudio/process_data/images_to_nerfstudio_dataset.py
+++ b/nerfstudio/process_data/images_to_nerfstudio_dataset.py
@@ -60,16 +60,24 @@ class ImagesToNerfstudioDataset(ColmapConverterToNerfstudioDataset):
if not self.skip_image_processing:
# Copy images to output directory
image_rename_map_paths = process_data_utils.copy_images(
- self.data, image_dir=self.image_dir, crop_factor=self.crop_factor, verbose=self.verbose
+ self.data,
+ image_dir=self.image_dir,
+ crop_factor=self.crop_factor,
+ verbose=self.verbose,
+ num_downscales=self.num_downscales,
)
image_rename_map = dict((a.name, b.name) for a, b in image_rename_map_paths.items())
num_frames = len(image_rename_map)
summary_log.append(f"Starting with {num_frames} images")
- # Downscale images
- summary_log.append(
- process_data_utils.downscale_images(self.image_dir, self.num_downscales, verbose=self.verbose)
+ # # Create mask
+ mask_path = process_data_utils.save_mask(
+ image_dir=self.image_dir,
+ num_downscales=self.num_downscales,
+ crop_factor=(0.0, 0.0, 0.0, 0.0),
)
+ if mask_path is not None:
+ summary_log.append("Saved mask(s)")
else:
num_frames = len(process_data_utils.list_images(self.data))
if num_frames == 0:
diff --git a/nerfstudio/process_data/polycam_utils.py b/nerfstudio/process_data/polycam_utils.py
index 416da15ad..0b1ba7a62 100644
--- a/nerfstudio/process_data/polycam_utils.py
+++ b/nerfstudio/process_data/polycam_utils.py
@@ -131,6 +131,7 @@ def process_images(
image_dir=image_dir,
crop_border_pixels=crop_border_pixels,
verbose=verbose,
+ num_downscales=num_downscales,
)
num_frames = len(copied_image_paths)
@@ -145,9 +146,6 @@ def process_images(
else:
summary_log.append(f"Started with {num_frames} images")
- # Downscale images
- summary_log.append(process_data_utils.downscale_images(image_dir, num_downscales, verbose=verbose))
-
# Save json
if num_frames == 0:
CONSOLE.print("[bold red]No images found, exiting")
@@ -190,7 +188,11 @@ def process_depth_maps(
# Copy depth images to output directory
copied_depth_maps_paths = process_data_utils.copy_and_upscale_polycam_depth_maps_list(
- polycam_depth_maps_filenames, depth_dir=depth_dir, crop_border_pixels=crop_border_pixels, verbose=verbose
+ polycam_depth_maps_filenames,
+ depth_dir=depth_dir,
+ num_downscales=num_downscales,
+ crop_border_pixels=crop_border_pixels,
+ verbose=verbose,
)
num_processed_depth_maps = len(copied_depth_maps_paths)
@@ -211,11 +213,4 @@ def process_depth_maps(
else:
summary_log.append(f"Started with {num_processed_depth_maps} images")
- # Downscale depth maps
- summary_log.append(
- process_data_utils.downscale_images(
- depth_dir, num_downscales, folder_name="depths", nearest_neighbor=True, verbose=verbose
- )
- )
-
return summary_log, polycam_depth_maps_filenames
diff --git a/nerfstudio/process_data/process_data_utils.py b/nerfstudio/process_data/process_data_utils.py
index 9bd835c3a..6d4cee094 100644
--- a/nerfstudio/process_data/process_data_utils.py
+++ b/nerfstudio/process_data/process_data_utils.py
@@ -111,6 +111,7 @@ def convert_video_to_images(
video_path: Path,
image_dir: Path,
num_frames_target: int,
+ num_downscales: int,
crop_factor: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0),
verbose: bool = False,
) -> Tuple[List[str], int]:
@@ -120,6 +121,7 @@ def convert_video_to_images(
video_path: Path to the video.
output_dir: Path to the output directory.
num_frames_target: Number of frames to extract.
+ num_downscales: Number of times to downscale the images. Downscales by 2 each time.
crop_factor: Portion of the image to crop. Should be in [0,1] (top, bottom, left, right)
verbose: If True, logs the output of the command.
Returns:
@@ -151,7 +153,6 @@ def convert_video_to_images(
sys.exit(1)
CONSOLE.print("Number of frames in video:", num_frames)
- out_filename = image_dir / "frame_%05d.png"
ffmpeg_cmd = f'ffmpeg -i "{video_path}"'
crop_cmd = ""
@@ -160,41 +161,66 @@ def convert_video_to_images(
width = 1 - crop_factor[2] - crop_factor[3]
start_x = crop_factor[2]
start_y = crop_factor[0]
- crop_cmd = f',"crop=w=iw*{width}:h=ih*{height}:x=iw*{start_x}:y=ih*{start_y}"'
+ crop_cmd = f"crop=w=iw*{width}:h=ih*{height}:x=iw*{start_x}:y=ih*{start_y},"
+ num_frames = get_num_frames_in_video(video_path)
spacing = num_frames // num_frames_target
+
+ downscale_chains = [f"[t{i}]scale=iw/{2**i}:ih/{2**i}[out{i}]" for i in range(num_downscales + 1)]
+ downscale_dirs = [Path(str(image_dir) + (f"_{2**i}" if i > 0 else "")) for i in range(num_downscales + 1)]
+ downscale_paths = [downscale_dirs[i] / "frame_%05d.png" for i in range(num_downscales + 1)]
+
+ for dir in downscale_dirs:
+ dir.mkdir(parents=True, exist_ok=True)
+
+ downscale_chain = (
+ f"split={num_downscales + 1}"
+ + "".join([f"[t{i}]" for i in range(num_downscales + 1)])
+ + ";"
+ + ";".join(downscale_chains)
+ )
+
if spacing > 1:
- ffmpeg_cmd += f" -vf thumbnail={spacing},setpts=N/TB{crop_cmd} -r 1"
CONSOLE.print("Number of frames to extract:", math.ceil(num_frames / spacing))
+ ffmpeg_cmd += " -vsync vfr -r1"
+ select_cmd = f"thumbnail={spacing},setpts=N/TB,"
else:
CONSOLE.print("[bold red]Can't satisfy requested number of frames. Extracting all frames.")
ffmpeg_cmd += " -pix_fmt bgr8"
- if crop_cmd != "":
- ffmpeg_cmd += f" -vf {crop_cmd[1:]}"
+ select_cmd = ""
+
+ downscale_cmd = f' -filter_complex "{select_cmd}{crop_cmd}{downscale_chain}"' + "".join(
+ [f' -map "[out{i}]" "{downscale_paths[i]}"' for i in range(num_downscales + 1)]
+ )
+
+ ffmpeg_cmd += downscale_cmd
- ffmpeg_cmd += f" {out_filename}"
run_command(ffmpeg_cmd, verbose=verbose)
- num_final_frames = len(list(image_dir.glob("*.png")))
- summary_log = []
- summary_log.append(f"Starting with {num_frames} video frames")
- summary_log.append(f"We extracted {num_final_frames} images")
- CONSOLE.log("[bold green]:tada: Done converting video to images.")
+ num_final_frames = len(list(image_dir.glob("*.png")))
+ summary_log = []
+ summary_log.append(f"Starting with {num_frames} video frames")
+ summary_log.append(f"We extracted {num_final_frames} images")
+ CONSOLE.log("[bold green]:tada: Done converting video to images.")
- return summary_log, num_final_frames
+ return summary_log, num_final_frames
def copy_images_list(
image_paths: List[Path],
image_dir: Path,
+ num_downscales: int,
crop_border_pixels: Optional[int] = None,
crop_factor: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0),
verbose: bool = False,
+ upscale_factor: Optional[int] = None,
+ nearest_neighbor: bool = False,
) -> List[Path]:
"""Copy all images in a list of Paths. Useful for filtering from a directory.
Args:
image_paths: List of Paths of images to copy to a new directory.
image_dir: Path to the output directory.
+ num_downscales: Number of times to downscale the images. Downscales by 2 each time.
crop_border_pixels: If not None, crops each edge by the specified number of pixels.
crop_factor: Portion of the image to crop. Should be in [0,1] (top, bottom, left, right)
verbose: If True, print extra logging.
@@ -230,26 +256,48 @@ def copy_images_list(
pass
copied_image_paths.append(copied_image_path)
+ nn_flag = "" if not nearest_neighbor else ":flags=neighbor"
+ downscale_chains = [f"[t{i}]scale=iw/{2**i}:ih/{2**i}{nn_flag}[out{i}]" for i in range(num_downscales + 1)]
+ downscale_dirs = [Path(str(image_dir) + (f"_{2**i}" if i > 0 else "")) for i in range(num_downscales + 1)]
+ downscale_paths = [
+ downscale_dirs[i] / ("frame_%05d" + copied_image_paths[0].suffix) for i in range(num_downscales + 1)
+ ]
+
+ for dir in downscale_dirs:
+ dir.mkdir(parents=True, exist_ok=True)
+
+ downscale_chain = (
+ f"split={num_downscales + 1}"
+ + "".join([f"[t{i}]" for i in range(num_downscales + 1)])
+ + ";"
+ + ";".join(downscale_chains)
+ )
+
+ num_frames = len(image_paths)
+ ffmpeg_cmd = f'ffmpeg -y -noautorotate -i "{image_dir / f"frame_%05d{copied_image_paths[0].suffix}"}" -q:v 2 '
+
+ crop_cmd = ""
if crop_border_pixels is not None:
- file_type = image_paths[0].suffix
- filename = f"frame_%05d{file_type}"
- crop = f"crop=iw-{crop_border_pixels*2}:ih-{crop_border_pixels*2}"
- ffmpeg_cmd = f'ffmpeg -y -noautorotate -i "{image_dir / filename}" -q:v 2 -vf {crop} "{image_dir / filename}"'
- run_command(ffmpeg_cmd, verbose=verbose)
+ crop_cmd = f"crop=iw-{crop_border_pixels*2}:ih-{crop_border_pixels*2}[cropped];[cropped]"
elif crop_factor != (0.0, 0.0, 0.0, 0.0):
- file_type = image_paths[0].suffix
- filename = f"frame_%05d{file_type}"
height = 1 - crop_factor[0] - crop_factor[1]
width = 1 - crop_factor[2] - crop_factor[3]
start_x = crop_factor[2]
start_y = crop_factor[0]
- crop_cmd = f',"crop=w=iw*{width}:h=ih*{height}:x=iw*{start_x}:y=ih*{start_y}"'
- ffmpeg_cmd = (
- f'ffmpeg -y -noautorotate -i "{image_dir / filename}" -q:v 2 -vf {crop_cmd[1:]} "{image_dir / filename}"'
- )
- run_command(ffmpeg_cmd, verbose=verbose)
+ crop_cmd = f"crop=w=iw*{width}:h=ih*{height}:x=iw*{start_x}:y=ih*{start_y}[cropped];[cropped]"
- num_frames = len(image_paths)
+ select_cmd = "[0:v]"
+ if upscale_factor is not None:
+ select_cmd = f"[0:v]scale=iw*{upscale_factor}:ih*{upscale_factor}:flags=neighbor[upscaled];[upscaled]"
+
+ downscale_cmd = f' -filter_complex "{select_cmd}{crop_cmd}{downscale_chain}"' + "".join(
+ [f' -map "[out{i}]" "{downscale_paths[i]}"' for i in range(num_downscales + 1)]
+ )
+
+ ffmpeg_cmd += downscale_cmd
+ if verbose:
+ CONSOLE.log(f"... {ffmpeg_cmd}")
+ run_command(ffmpeg_cmd, verbose=verbose)
if num_frames == 0:
CONSOLE.log("[bold red]:skull: No usable images in the data folder.")
@@ -262,6 +310,7 @@ def copy_images_list(
def copy_and_upscale_polycam_depth_maps_list(
polycam_depth_image_filenames: List[Path],
depth_dir: Path,
+ num_downscales: int,
crop_border_pixels: Optional[int] = None,
verbose: bool = False,
) -> List[Path]:
@@ -284,31 +333,26 @@ def copy_and_upscale_polycam_depth_maps_list(
assert upscale_factor > 1
assert isinstance(upscale_factor, int)
- copied_depth_map_paths = []
- for idx, depth_map in enumerate(polycam_depth_image_filenames):
- destination = depth_dir / f"frame_{idx + 1:05d}{depth_map.suffix}"
- ffmpeg_cmd = [
- f'ffmpeg -y -i "{depth_map}" ',
- f"-q:v 2 -vf scale=iw*{upscale_factor}:ih*{upscale_factor}:flags=neighbor ",
- f'"{destination}"',
- ]
- ffmpeg_cmd = " ".join(ffmpeg_cmd)
- run_command(ffmpeg_cmd, verbose=verbose)
- copied_depth_map_paths.append(destination)
-
- if crop_border_pixels is not None:
- file_type = depth_dir.glob("frame_*").__next__().suffix
- filename = f"frame_%05d{file_type}"
- crop = f"crop=iw-{crop_border_pixels * 2}:ih-{crop_border_pixels * 2}"
- ffmpeg_cmd = f'ffmpeg -y -i "{depth_dir / filename}" -q:v 2 -vf {crop} "{depth_dir / filename}"'
- run_command(ffmpeg_cmd, verbose=verbose)
+ copied_depth_map_paths = copy_images_list(
+ image_paths=polycam_depth_image_filenames,
+ image_dir=depth_dir,
+ num_downscales=num_downscales,
+ crop_border_pixels=crop_border_pixels,
+ verbose=verbose,
+ upscale_factor=upscale_factor,
+ nearest_neighbor=True,
+ )
CONSOLE.log("[bold green]:tada: Done upscaling depth maps.")
return copied_depth_map_paths
def copy_images(
- data: Path, image_dir: Path, verbose, crop_factor: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0)
+ data: Path,
+ image_dir: Path,
+ verbose: bool = False,
+ crop_factor: Tuple[float, float, float, float] = (0.0, 0.0, 0.0, 0.0),
+ num_downscales: int = 0,
) -> OrderedDict[Path, Path]:
"""Copy images from a directory to a new directory.
@@ -328,7 +372,11 @@ def copy_images(
sys.exit(1)
copied_images = copy_images_list(
- image_paths=image_paths, image_dir=image_dir, crop_factor=crop_factor, verbose=verbose
+ image_paths=image_paths,
+ image_dir=image_dir,
+ crop_factor=crop_factor,
+ verbose=verbose,
+ num_downscales=num_downscales,
)
return OrderedDict((original_path, new_path) for original_path, new_path in zip(image_paths, copied_images))
@@ -340,7 +388,8 @@ def downscale_images(
nearest_neighbor: bool = False,
verbose: bool = False,
) -> str:
- """Downscales the images in the directory. Uses FFMPEG.
+ """(Now deprecated; much faster integrated into copy_images.)
+ Downscales the images in the directory. Uses FFMPEG.
Assumes images are named frame_00001.png, frame_00002.png, etc.
diff --git a/nerfstudio/process_data/video_to_nerfstudio_dataset.py b/nerfstudio/process_data/video_to_nerfstudio_dataset.py
index 13e4b6bc4..b1eec97ea 100644
--- a/nerfstudio/process_data/video_to_nerfstudio_dataset.py
+++ b/nerfstudio/process_data/video_to_nerfstudio_dataset.py
@@ -30,9 +30,8 @@ class VideoToNerfstudioDataset(ColmapConverterToNerfstudioDataset):
This script does the following:
- 1. Converts the video into images.
- 2. Scales images to a specified size.
- 3. Calculates the camera poses for each image using `COLMAP `_.
+ 1. Converts the video into images and downscales them.
+ 2. Calculates the camera poses for each image using `COLMAP `_.
"""
num_frames_target: int = 300
@@ -53,14 +52,17 @@ class VideoToNerfstudioDataset(ColmapConverterToNerfstudioDataset):
self.data,
image_dir=temp_image_dir,
num_frames_target=self.num_frames_target,
+ num_downscales=0,
crop_factor=(0.0, 0.0, 0.0, 0.0),
verbose=self.verbose,
)
else:
+ # If we're not dealing with equirects we can downscale in one step.
summary_log, num_extracted_frames = process_data_utils.convert_video_to_images(
self.data,
image_dir=self.image_dir,
num_frames_target=self.num_frames_target,
+ num_downscales=self.num_downscales,
crop_factor=self.crop_factor,
verbose=self.verbose,
)
@@ -90,6 +92,11 @@ class VideoToNerfstudioDataset(ColmapConverterToNerfstudioDataset):
self.camera_type = "perspective"
+ # # Downscale images
+ summary_log.append(
+ process_data_utils.downscale_images(self.image_dir, self.num_downscales, verbose=self.verbose)
+ )
+
# # Create mask
mask_path = process_data_utils.save_mask(
image_dir=self.image_dir,
@@ -100,11 +107,6 @@ class VideoToNerfstudioDataset(ColmapConverterToNerfstudioDataset):
if mask_path is not None:
summary_log.append(f"Saved mask to {mask_path}")
- # # Downscale images
- summary_log.append(
- process_data_utils.downscale_images(self.image_dir, self.num_downscales, verbose=self.verbose)
- )
-
# Run Colmap
if not self.skip_colmap:
self._run_colmap(mask_path)
diff --git a/nerfstudio/scripts/process_data.py b/nerfstudio/scripts/process_data.py
index 50a10bb5a..201eb94fc 100644
--- a/nerfstudio/scripts/process_data.py
+++ b/nerfstudio/scripts/process_data.py
@@ -89,7 +89,10 @@ class ProcessRecord3D(BaseConverterToNerfstudioDataset):
record3d_image_filenames = list(np.array(record3d_image_filenames)[idx])
# Copy images to output directory
copied_image_paths = process_data_utils.copy_images_list(
- record3d_image_filenames, image_dir=image_dir, verbose=self.verbose
+ record3d_image_filenames,
+ image_dir=image_dir,
+ verbose=self.verbose,
+ num_downscales=self.num_downscales,
)
num_frames = len(copied_image_paths)
@@ -101,9 +104,6 @@ class ProcessRecord3D(BaseConverterToNerfstudioDataset):
f"larger than the current value ({self.max_dataset_size}), or -1 to use all images."
)
- # Downscale images
- summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose))
-
metadata_path = self.data / "metadata.json"
record3d_utils.record3d_to_json(copied_image_paths, metadata_path, self.output_dir, indices=idx)
CONSOLE.rule("[bold green]:tada: :tada: :tada: All DONE :tada: :tada: :tada:")
@@ -266,6 +266,7 @@ class ProcessMetashape(BaseConverterToNerfstudioDataset, _NoDefaultProcessMetash
image_filenames,
image_dir=image_dir,
verbose=self.verbose,
+ num_downscales=self.num_downscales,
)
num_frames = len(copied_image_paths)
@@ -282,9 +283,6 @@ class ProcessMetashape(BaseConverterToNerfstudioDataset, _NoDefaultProcessMetash
else:
summary_log.append(f"Started with {num_frames} images")
- # Downscale images
- summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose))
-
# Save json
if num_frames == 0:
CONSOLE.print("[bold red]No images found, exiting")
@@ -353,6 +351,7 @@ class ProcessRealityCapture(BaseConverterToNerfstudioDataset, _NoDefaultProcessR
image_filenames,
image_dir=image_dir,
verbose=self.verbose,
+ num_downscales=self.num_downscales,
)
num_frames = len(copied_image_paths)
@@ -369,9 +368,6 @@ class ProcessRealityCapture(BaseConverterToNerfstudioDataset, _NoDefaultProcessR
else:
summary_log.append(f"Started with {num_frames} images")
- # Downscale images
- summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose))
-
# Save json
if num_frames == 0:
CONSOLE.print("[bold red]No images found, exiting")