From 52cda21186a9a50b29e7480d3dec20b5b15323d3 Mon Sep 17 00:00:00 2001 From: Congrong Xu <50019703+KevinXu02@users.noreply.github.com> Date: Mon, 5 Feb 2024 09:04:14 -0800 Subject: [PATCH] Add multi-processing support for cache_image. Fixed the issue of insufficient precision leading to a completely black image when converting from RGBA to RGB. (#2867) --- .../datamanagers/full_images_datamanager.py | 98 ++++++++++++------- nerfstudio/data/datasets/base_dataset.py | 2 +- 2 files changed, 61 insertions(+), 39 deletions(-) diff --git a/nerfstudio/data/datamanagers/full_images_datamanager.py b/nerfstudio/data/datamanagers/full_images_datamanager.py index d362467d3..9b2057357 100644 --- a/nerfstudio/data/datamanagers/full_images_datamanager.py +++ b/nerfstudio/data/datamanagers/full_images_datamanager.py @@ -22,6 +22,7 @@ paradigm from __future__ import annotations import random +from concurrent.futures import ThreadPoolExecutor from copy import deepcopy from dataclasses import dataclass, field from functools import cached_property @@ -31,8 +32,8 @@ from typing import Dict, ForwardRef, Generic, List, Literal, Optional, Tuple, Ty import cv2 import numpy as np import torch +from rich.progress import track from torch.nn import Parameter -from tqdm import tqdm from nerfstudio.cameras.camera_utils import fisheye624_project, fisheye624_unproject_helper from nerfstudio.cameras.cameras import Cameras, CameraType @@ -64,6 +65,8 @@ class FullImageDatamanagerConfig(DataManagerConfig): """Whether to cache images in memory. If "cpu", caches on cpu. If "gpu", caches on device.""" cache_images_type: Literal["uint8", "float32"] = "float32" """The image type returned from manager, caching images in uint8 saves memory""" + max_thread_workers: Optional[int] = None + """The maximum number of threads to use for caching images. If None, uses all available threads.""" class FullImageDatamanager(DataManager, Generic[TDataset]): @@ -129,15 +132,14 @@ class FullImageDatamanager(DataManager, Generic[TDataset]): def cache_images(self, cache_images_option): cached_train = [] cached_eval = [] - CONSOLE.log("Caching / undistorting train images") - for i in tqdm(range(len(self.train_dataset)), leave=False): + + def process_train_data(idx): # cv2.undistort the images / cameras - data = self.train_dataset.get_data(i, image_type=self.config.cache_images_type) - camera = self.train_dataset.cameras[i].reshape(()) + data = self.train_dataset.get_data(idx, image_type=self.config.cache_images_type) + camera = self.train_dataset.cameras[idx].reshape(()) K = camera.get_intrinsics_matrices().numpy() if camera.distortion_params is None: - cached_train.append(data) - continue + return data distortion_params = camera.distortion_params.numpy() image = data["image"].numpy() @@ -146,40 +148,60 @@ class FullImageDatamanager(DataManager, Generic[TDataset]): if mask is not None: data["mask"] = mask - cached_train.append(data) + self.train_dataset.cameras.fx[idx] = float(K[0, 0]) + self.train_dataset.cameras.fy[idx] = float(K[1, 1]) + self.train_dataset.cameras.cx[idx] = float(K[0, 2]) + self.train_dataset.cameras.cy[idx] = float(K[1, 2]) + self.train_dataset.cameras.width[idx] = image.shape[1] + self.train_dataset.cameras.height[idx] = image.shape[0] + return data - self.train_dataset.cameras.fx[i] = float(K[0, 0]) - self.train_dataset.cameras.fy[i] = float(K[1, 1]) - self.train_dataset.cameras.cx[i] = float(K[0, 2]) - self.train_dataset.cameras.cy[i] = float(K[1, 2]) - self.train_dataset.cameras.width[i] = image.shape[1] - self.train_dataset.cameras.height[i] = image.shape[0] + def process_eval_data(idx): + # cv2.undistort the images / cameras + data = self.eval_dataset.get_data(idx, image_type=self.config.cache_images_type) + camera = self.eval_dataset.cameras[idx].reshape(()) + K = camera.get_intrinsics_matrices().numpy() + if camera.distortion_params is None: + return data + distortion_params = camera.distortion_params.numpy() + image = data["image"].numpy() + + K, image, mask = _undistort_image(camera, distortion_params, data, image, K) + data["image"] = torch.from_numpy(image) + if mask is not None: + data["mask"] = mask + + self.eval_dataset.cameras.fx[idx] = float(K[0, 0]) + self.eval_dataset.cameras.fy[idx] = float(K[1, 1]) + self.eval_dataset.cameras.cx[idx] = float(K[0, 2]) + self.eval_dataset.cameras.cy[idx] = float(K[1, 2]) + self.eval_dataset.cameras.width[idx] = image.shape[1] + self.eval_dataset.cameras.height[idx] = image.shape[0] + return data + + CONSOLE.log("Caching / undistorting train images") + with ThreadPoolExecutor() as executor: + cached_train = list( + executor.map( + process_train_data, + track( + range(len(self.train_dataset)), + description="Caching / undistorting train images", + transient=True, + ), + ) + ) CONSOLE.log("Caching / undistorting eval images") - for i in tqdm(range(len(self.eval_dataset)), leave=False): - # cv2.undistort the images / cameras - data = self.eval_dataset.get_data(i, image_type=self.config.cache_images_type) - camera = self.eval_dataset.cameras[i].reshape(()) - K = camera.get_intrinsics_matrices().numpy() - if camera.distortion_params is None: - cached_eval.append(data) - continue - distortion_params = camera.distortion_params.numpy() - image = data["image"].numpy() - - K, image, mask = _undistort_image(camera, distortion_params, data, image, K) - data["image"] = torch.from_numpy(image) - if mask is not None: - data["mask"] = mask - - cached_eval.append(data) - - self.eval_dataset.cameras.fx[i] = float(K[0, 0]) - self.eval_dataset.cameras.fy[i] = float(K[1, 1]) - self.eval_dataset.cameras.cx[i] = float(K[0, 2]) - self.eval_dataset.cameras.cy[i] = float(K[1, 2]) - self.eval_dataset.cameras.width[i] = image.shape[1] - self.eval_dataset.cameras.height[i] = image.shape[0] + with ThreadPoolExecutor() as executor: + cached_eval = list( + executor.map( + process_eval_data, + track( + range(len(self.eval_dataset)), description="Caching / undistorting eval images", transient=True + ), + ) + ) if cache_images_option == "gpu": for cache in cached_train: diff --git a/nerfstudio/data/datasets/base_dataset.py b/nerfstudio/data/datasets/base_dataset.py index 5d44470fe..5e697375b 100644 --- a/nerfstudio/data/datasets/base_dataset.py +++ b/nerfstudio/data/datasets/base_dataset.py @@ -103,7 +103,7 @@ class InputDataset(Dataset): assert (self._dataparser_outputs.alpha_color >= 0).all() and ( self._dataparser_outputs.alpha_color <= 1 ).all(), "alpha color given is out of range between [0, 1]." - image = image[:, :, :3] * image[:, :, -1:] / 255.0 + 255.0 * self._dataparser_outputs.alpha_color * ( + image = image[:, :, :3] * (image[:, :, -1:] / 255.0) + 255.0 * self._dataparser_outputs.alpha_color * ( 1.0 - image[:, :, -1:] / 255.0 ) image = torch.clamp(image, min=0, max=255).to(torch.uint8)