Add multi-processing support for cache_image. Fixed the issue of insufficient precision leading to a completely black image when converting from RGBA to RGB. (#2867)

This commit is contained in:
Congrong Xu authored and GitHub committed 2024-02-05 17:04:14 +00:00
1 parent 2046cfe9fb
commit 52cda21186
2 files changed
+61 -39

No files matched your search

@@ -22,6 +22,7 @@ paradigm
from __future__ import annotations
import random
from concurrent.futures import ThreadPoolExecutor
from copy import deepcopy
from dataclasses import dataclass, field
from functools import cached_property
@@ -31,8 +32,8 @@ from typing import Dict, ForwardRef, Generic, List, Literal, Optional, Tuple, Ty
import cv2
import numpy as np
import torch
from rich.progress import track
from torch.nn import Parameter
from tqdm import tqdm
from nerfstudio.cameras.camera_utils import fisheye624_project, fisheye624_unproject_helper
from nerfstudio.cameras.cameras import Cameras, CameraType
@@ -64,6 +65,8 @@ class FullImageDatamanagerConfig(DataManagerConfig):
"""Whether to cache images in memory. If "cpu", caches on cpu. If "gpu", caches on device."""
cache_images_type: Literal["uint8", "float32"] = "float32"
"""The image type returned from manager, caching images in uint8 saves memory"""
max_thread_workers: Optional[int] = None
"""The maximum number of threads to use for caching images. If None, uses all available threads."""
class FullImageDatamanager(DataManager, Generic[TDataset]):
@@ -129,15 +132,14 @@ class FullImageDatamanager(DataManager, Generic[TDataset]):
def cache_images(self, cache_images_option):
cached_train = []
cached_eval = []
CONSOLE.log("Caching / undistorting train images")
for i in tqdm(range(len(self.train_dataset)), leave=False):
def process_train_data(idx):
# cv2.undistort the images / cameras
data = self.train_dataset.get_data(i, image_type=self.config.cache_images_type)
camera = self.train_dataset.cameras[i].reshape(())
data = self.train_dataset.get_data(idx, image_type=self.config.cache_images_type)
camera = self.train_dataset.cameras[idx].reshape(())
K = camera.get_intrinsics_matrices().numpy()
if camera.distortion_params is None:
cached_train.append(data)
continue
return data
distortion_params = camera.distortion_params.numpy()
image = data["image"].numpy()
@@ -146,40 +148,60 @@ class FullImageDatamanager(DataManager, Generic[TDataset]):
if mask is not None:
data["mask"] = mask
cached_train.append(data)
self.train_dataset.cameras.fx[idx] = float(K[0, 0])
self.train_dataset.cameras.fy[idx] = float(K[1, 1])
self.train_dataset.cameras.cx[idx] = float(K[0, 2])
self.train_dataset.cameras.cy[idx] = float(K[1, 2])
self.train_dataset.cameras.width[idx] = image.shape[1]
self.train_dataset.cameras.height[idx] = image.shape[0]
return data
self.train_dataset.cameras.fx[i] = float(K[0, 0])
self.train_dataset.cameras.fy[i] = float(K[1, 1])
self.train_dataset.cameras.cx[i] = float(K[0, 2])
self.train_dataset.cameras.cy[i] = float(K[1, 2])
self.train_dataset.cameras.width[i] = image.shape[1]
self.train_dataset.cameras.height[i] = image.shape[0]
def process_eval_data(idx):
# cv2.undistort the images / cameras
data = self.eval_dataset.get_data(idx, image_type=self.config.cache_images_type)
camera = self.eval_dataset.cameras[idx].reshape(())
K = camera.get_intrinsics_matrices().numpy()
if camera.distortion_params is None:
return data
distortion_params = camera.distortion_params.numpy()
image = data["image"].numpy()
K, image, mask = _undistort_image(camera, distortion_params, data, image, K)
data["image"] = torch.from_numpy(image)
if mask is not None:
data["mask"] = mask
self.eval_dataset.cameras.fx[idx] = float(K[0, 0])
self.eval_dataset.cameras.fy[idx] = float(K[1, 1])
self.eval_dataset.cameras.cx[idx] = float(K[0, 2])
self.eval_dataset.cameras.cy[idx] = float(K[1, 2])
self.eval_dataset.cameras.width[idx] = image.shape[1]
self.eval_dataset.cameras.height[idx] = image.shape[0]
return data
CONSOLE.log("Caching / undistorting train images")
with ThreadPoolExecutor() as executor:
cached_train = list(
executor.map(
process_train_data,
track(
range(len(self.train_dataset)),
description="Caching / undistorting train images",
transient=True,
),
)
)
CONSOLE.log("Caching / undistorting eval images")
for i in tqdm(range(len(self.eval_dataset)), leave=False):
# cv2.undistort the images / cameras
data = self.eval_dataset.get_data(i, image_type=self.config.cache_images_type)
camera = self.eval_dataset.cameras[i].reshape(())
K = camera.get_intrinsics_matrices().numpy()
if camera.distortion_params is None:
cached_eval.append(data)
continue
distortion_params = camera.distortion_params.numpy()
image = data["image"].numpy()
K, image, mask = _undistort_image(camera, distortion_params, data, image, K)
data["image"] = torch.from_numpy(image)
if mask is not None:
data["mask"] = mask
cached_eval.append(data)
self.eval_dataset.cameras.fx[i] = float(K[0, 0])
self.eval_dataset.cameras.fy[i] = float(K[1, 1])
self.eval_dataset.cameras.cx[i] = float(K[0, 2])
self.eval_dataset.cameras.cy[i] = float(K[1, 2])
self.eval_dataset.cameras.width[i] = image.shape[1]
self.eval_dataset.cameras.height[i] = image.shape[0]
with ThreadPoolExecutor() as executor:
cached_eval = list(
executor.map(
process_eval_data,
track(
range(len(self.eval_dataset)), description="Caching / undistorting eval images", transient=True
),
)
)
if cache_images_option == "gpu":
for cache in cached_train:
+1 -1
View File
@@ -103,7 +103,7 @@ class InputDataset(Dataset):
assert (self._dataparser_outputs.alpha_color >= 0).all() and (
self._dataparser_outputs.alpha_color <= 1
).all(), "alpha color given is out of range between [0, 1]."
image = image[:, :, :3] * image[:, :, -1:] / 255.0 + 255.0 * self._dataparser_outputs.alpha_color * (
image = image[:, :, :3] * (image[:, :, -1:] / 255.0) + 255.0 * self._dataparser_outputs.alpha_color * (
1.0 - image[:, :, -1:] / 255.0
)
image = torch.clamp(image, min=0, max=255).to(torch.uint8)