mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-20 14:45:38 +08:00
More typing fixes (#1941)
* Fix more types * Fix tests * Minor cleaning * Address comments * Resolve linting errors * Re-ignore pymeshlab type * Add extra assert / error message to mipnerf model
This commit is contained in:
1 parent
8b9574bb5a
commit
814db32ec4
53 files changed
+386
-205
No files matched your search
@@ -22,7 +22,18 @@ import functools
|
||||
from abc import abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Generic, List, Literal, Optional, Tuple, Type, Union
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
Dict,
|
||||
Generic,
|
||||
List,
|
||||
Literal,
|
||||
Optional,
|
||||
Tuple,
|
||||
Type,
|
||||
Union,
|
||||
)
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
@@ -72,7 +83,7 @@ def variable_res_collate(batch: List[Dict]) -> Dict:
|
||||
if mask is not None:
|
||||
masks.append(mask)
|
||||
|
||||
new_batch: dict = nerfstudio_collate(batch)
|
||||
new_batch = nerfstudio_collate(batch)
|
||||
new_batch["image"] = images
|
||||
if masks:
|
||||
new_batch["mask"] = masks
|
||||
@@ -314,7 +325,7 @@ class VanillaDataManagerConfig(DataManagerConfig):
|
||||
camera_optimizer: CameraOptimizerConfig = CameraOptimizerConfig()
|
||||
"""Specifies the camera pose optimizer used during training. Helpful if poses are noisy, such as for data from
|
||||
Record3D."""
|
||||
collate_fn = staticmethod(nerfstudio_collate)
|
||||
collate_fn: Callable[[Any], Any] = staticmethod(nerfstudio_collate)
|
||||
"""Specifies the collate function to use for the train and eval dataloaders."""
|
||||
camera_res_scale_factor: float = 1.0
|
||||
"""The scale factor for scaling spatial data such as images, mask, semantics
|
||||
@@ -356,7 +367,7 @@ class VanillaDataManager(DataManager, Generic[TDataset]): # pylint: disable=abs
|
||||
local_rank: int = 0,
|
||||
**kwargs, # pylint: disable=unused-argument
|
||||
):
|
||||
self.dataset_type: Type[TDataset] = kwargs.get("_dataset_type", TDataset.__default__)
|
||||
self.dataset_type: Type[TDataset] = kwargs.get("_dataset_type", getattr(TDataset, "__default__"))
|
||||
self.config = config
|
||||
self.device = device
|
||||
self.world_size = world_size
|
||||
@@ -486,6 +497,7 @@ class VanillaDataManager(DataManager, Generic[TDataset]): # pylint: disable=abs
|
||||
self.train_count += 1
|
||||
image_batch = next(self.iter_train_image_dataloader)
|
||||
assert self.train_pixel_sampler is not None
|
||||
assert isinstance(image_batch, dict)
|
||||
batch = self.train_pixel_sampler.sample(image_batch)
|
||||
ray_indices = batch["indices"]
|
||||
ray_bundle = self.train_ray_generator(ray_indices)
|
||||
@@ -496,6 +508,7 @@ class VanillaDataManager(DataManager, Generic[TDataset]): # pylint: disable=abs
|
||||
self.eval_count += 1
|
||||
image_batch = next(self.iter_eval_image_dataloader)
|
||||
assert self.eval_pixel_sampler is not None
|
||||
assert isinstance(image_batch, dict)
|
||||
batch = self.eval_pixel_sampler.sample(image_batch)
|
||||
ray_indices = batch["indices"]
|
||||
ray_bundle = self.eval_ray_generator(ray_indices)
|
||||
|
||||
@@ -50,7 +50,7 @@ def traj_string_to_matrix(traj_string: str):
|
||||
ts = tokens[0]
|
||||
# Rotation in angle axis
|
||||
angle_axis = [float(tokens[1]), float(tokens[2]), float(tokens[3])]
|
||||
r_w_to_p, _ = cv2.Rodrigues(np.asarray(angle_axis))
|
||||
r_w_to_p, _ = cv2.Rodrigues(np.asarray(angle_axis)) # type: ignore
|
||||
# Translation
|
||||
t_w_to_p = np.asarray([float(tokens[4]), float(tokens[5]), float(tokens[6])])
|
||||
extrinsics = np.eye(4, 4)
|
||||
@@ -213,6 +213,7 @@ class ARKitScenes(DataParser):
|
||||
|
||||
@staticmethod
|
||||
def _get_pose(frame_id: str, poses_from_traj: dict):
|
||||
frame_pose = None
|
||||
if str(frame_id) in poses_from_traj:
|
||||
frame_pose = np.array(poses_from_traj[str(frame_id)])
|
||||
else:
|
||||
@@ -220,6 +221,7 @@ class ARKitScenes(DataParser):
|
||||
if abs(float(frame_id) - float(my_key)) < 0.005:
|
||||
frame_pose = np.array(poses_from_traj[str(my_key)])
|
||||
|
||||
assert frame_pose is not None
|
||||
frame_pose[0:3, 1:3] *= -1
|
||||
frame_pose = frame_pose[np.array([1, 0, 2, 3]), :]
|
||||
frame_pose[2, :] *= -1
|
||||
|
||||
@@ -52,7 +52,7 @@ def downscale(img, scale: int) -> np.ndarray:
|
||||
if height % scale > 0 or width % scale > 0:
|
||||
raise ValueError(f"Image shape ({height},{width}) must be divisible by the" f" scale ({scale}).")
|
||||
out_height, out_width = height // scale, width // scale
|
||||
resized = cv2.resize(img, (out_width, out_height), cv2.INTER_AREA)
|
||||
resized = cv2.resize(img, (out_width, out_height), cv2.INTER_AREA) # type: ignore
|
||||
return resized
|
||||
|
||||
|
||||
@@ -70,11 +70,11 @@ def upscale(img, scale: int) -> np.ndarray:
|
||||
return img
|
||||
height, width = img.shape[:2]
|
||||
out_height, out_width = height * scale, width * scale
|
||||
resized = cv2.resize(img, (out_width, out_height), cv2.INTER_AREA)
|
||||
resized = cv2.resize(img, (out_width, out_height), cv2.INTER_AREA) # type: ignore
|
||||
return resized
|
||||
|
||||
|
||||
def rescale(img, scale_factor: float, interpolation=cv2.INTER_AREA) -> np.ndarray:
|
||||
def rescale(img, scale_factor: float, interpolation=cv2.INTER_AREA) -> np.ndarray: # type: ignore
|
||||
"""Function from DyCheck's repo. Rescale an image.
|
||||
|
||||
Args:
|
||||
@@ -106,7 +106,7 @@ def rescale(img, scale_factor: float, interpolation=cv2.INTER_AREA) -> np.ndarra
|
||||
out_width = math.ceil(width * scale_factor)
|
||||
out_width -= out_width % 2
|
||||
|
||||
return cv2.resize(img, (out_width, out_height), interpolation)
|
||||
return cv2.resize(img, (out_width, out_height), interpolation) # type: ignore
|
||||
|
||||
|
||||
def _load_scene_info(data_dir: Path) -> Tuple[np.ndarray, float, float, float]:
|
||||
@@ -242,7 +242,7 @@ class Dycheck(DataParser):
|
||||
frame_names = np.array(split_dict["frame_names"])[[0]]
|
||||
time_ids = np.array(split_dict["time_ids"])[[0]]
|
||||
|
||||
image_filenames, depth_filenames, cams = self.process_frames(frame_names, time_ids)
|
||||
image_filenames, depth_filenames, cams = self.process_frames(frame_names.tolist(), time_ids)
|
||||
|
||||
scene_box = SceneBox(
|
||||
aabb=torch.tensor(
|
||||
@@ -316,13 +316,13 @@ class Dycheck(DataParser):
|
||||
d = self.config.downscale_factor
|
||||
if not image_filenames[0].exists():
|
||||
CONSOLE.print(f"downscale factor {d}x not exist, converting")
|
||||
ori_h, ori_w = cv2.imread(str(self.data / f"rgb/1x/{frame_names[0]}.png")).shape[:2]
|
||||
ori_h, ori_w = cv2.imread(str(self.data / f"rgb/1x/{frame_names[0]}.png")).shape[:2] # type: ignore
|
||||
(self.data / f"rgb/{d}x").mkdir(exist_ok=True)
|
||||
h, w = ori_h // d, ori_w // d
|
||||
for frame in frame_names:
|
||||
cv2.imwrite(
|
||||
cv2.imwrite( # type: ignore
|
||||
str(self.data / f"rgb/{d}x/{frame}.png"),
|
||||
cv2.resize(cv2.imread(str(self.data / f"rgb/1x/{frame}.png")), [h, w]),
|
||||
cv2.resize(cv2.imread(str(self.data / f"rgb/1x/{frame}.png")), [h, w]), # type: ignore
|
||||
)
|
||||
CONSOLE.print("finished")
|
||||
|
||||
@@ -331,8 +331,8 @@ class Dycheck(DataParser):
|
||||
(self.data / f"processed_depth/{d}x").mkdir(exist_ok=True, parents=True)
|
||||
for idx, frame in enumerate(frame_names):
|
||||
depth = np.load(self.data / f"depth/1x/{frame}.npy")
|
||||
mask = rescale((depth != 0).astype(np.uint8) * 255, 1 / d, cv2.INTER_AREA)
|
||||
depth = rescale(depth, 1 / d, cv2.INTER_AREA)
|
||||
mask = rescale((depth != 0).astype(np.uint8) * 255, 1 / d, cv2.INTER_AREA) # type: ignore
|
||||
depth = rescale(depth, 1 / d, cv2.INTER_AREA) # type: ignore
|
||||
depth[mask != 255] = 0
|
||||
depth = _rescale_depth(depth, cams[idx])
|
||||
np.save(str(self.data / f"processed_depth/{d}x/{frame}.npy"), depth)
|
||||
|
||||
@@ -57,7 +57,6 @@ class MinimalDataParser(DataParser):
|
||||
if "mask_filenames" in data.keys():
|
||||
mask_filenames = [filepath.parent / path for path in data["mask_filenames"].tolist()]
|
||||
|
||||
metadata = None
|
||||
if "semantics" in data.keys():
|
||||
semantics = data["semantics"].item()
|
||||
metadata = {
|
||||
@@ -68,6 +67,8 @@ class MinimalDataParser(DataParser):
|
||||
mask_classes=semantics["mask_classes"].tolist(),
|
||||
)
|
||||
}
|
||||
else:
|
||||
metadata = {}
|
||||
|
||||
scene_box_aabb = torch.from_numpy(data["scene_box"])
|
||||
scene_box = SceneBox(aabb=scene_box_aabb)
|
||||
@@ -90,7 +91,7 @@ class MinimalDataParser(DataParser):
|
||||
)
|
||||
|
||||
applied_scale = 1.0
|
||||
applied_transform = np.eye(4, dtype=np.float32)[:3, :]
|
||||
applied_transform = torch.eye(4, dtype=torch.float32)[:3, :]
|
||||
if "applied_scale" in data.keys():
|
||||
applied_scale = float(data["applied_scale"])
|
||||
if "applied_transform" in data.keys():
|
||||
|
||||
@@ -87,7 +87,11 @@ class NuScenes(DataParser):
|
||||
def _generate_dataparser_outputs(self, split="train"):
|
||||
# pylint: disable=too-many-statements
|
||||
|
||||
nusc = NuScenesDatabase(version=self.config.version, dataroot=self.config.data_dir, verbose=self.config.verbose)
|
||||
nusc = NuScenesDatabase(
|
||||
version=self.config.version,
|
||||
dataroot=str(self.config.data_dir.absolute()),
|
||||
verbose=self.config.verbose,
|
||||
)
|
||||
cameras = ["CAM_" + camera for camera in self.config.cameras]
|
||||
|
||||
assert (
|
||||
|
||||
@@ -81,7 +81,7 @@ class ScanNet(DataParser):
|
||||
depth_dir_sorted = list(sorted(depth_dir.iterdir(), key=lambda x: int(x.name.split(".")[0])))
|
||||
pose_dir_sorted = list(sorted(pose_dir.iterdir(), key=lambda x: int(x.name.split(".")[0])))
|
||||
|
||||
first_img = cv2.imread(img_dir_sorted[0].as_posix())
|
||||
first_img = cv2.imread(str(img_dir_sorted[0].absolute())) # type: ignore
|
||||
h, w, _ = first_img.shape
|
||||
|
||||
image_filenames, depth_filenames, intrinsics, poses = [], [], [], []
|
||||
|
||||
@@ -111,11 +111,12 @@ class Sitcoms3D(DataParser):
|
||||
lengths = scene_box.aabb[1] - scene_box.aabb[0]
|
||||
longest_dim = torch.argmax(lengths)
|
||||
longest_length = lengths[longest_dim]
|
||||
scale = scene_scale / longest_length
|
||||
scale = scene_scale / longest_length.item()
|
||||
scene_box.aabb = scene_box.aabb * scale # box
|
||||
camera_to_worlds[..., 3] *= scale # cameras
|
||||
|
||||
# --- semantics ---
|
||||
semantics = None
|
||||
if self.config.include_semantics:
|
||||
empty_path = Path()
|
||||
replace_this_path = str(empty_path / images_folder / empty_path)
|
||||
|
||||
@@ -44,7 +44,6 @@ class InputDataset(Dataset):
|
||||
def __init__(self, dataparser_outputs: DataparserOutputs, scale_factor: float = 1.0):
|
||||
super().__init__()
|
||||
self._dataparser_outputs = dataparser_outputs
|
||||
self.has_masks = dataparser_outputs.mask_filenames is not None
|
||||
self.scale_factor = scale_factor
|
||||
self.scene_box = deepcopy(dataparser_outputs.scene_box)
|
||||
self.metadata = deepcopy(dataparser_outputs.metadata)
|
||||
@@ -95,9 +94,8 @@ class InputDataset(Dataset):
|
||||
image_idx: The image index in the dataset.
|
||||
"""
|
||||
image = self.get_image(image_idx)
|
||||
data = {"image_idx": image_idx}
|
||||
data["image"] = image
|
||||
if self.has_masks:
|
||||
data = {"image_idx": image_idx, "image": image}
|
||||
if self._dataparser_outputs.mask_filenames is not None:
|
||||
mask_filepath = self._dataparser_outputs.mask_filenames[image_idx]
|
||||
data["mask"] = get_image_mask_tensor_from_path(filepath=mask_filepath, scale_factor=self.scale_factor)
|
||||
assert (
|
||||
|
||||
@@ -81,7 +81,7 @@ class SDFDataset(InputDataset):
|
||||
normal = normal * 2.0 - 1.0 # omnidata output is normalized so we convert it back to normal here
|
||||
normal = torch.from_numpy(normal).float()
|
||||
|
||||
rot = camtoworld[:3, :3]
|
||||
rot = torch.from_numpy(camtoworld[:3, :3])
|
||||
|
||||
normal_map = normal.reshape(3, -1)
|
||||
normal_map = torch.nn.functional.normalize(normal_map, p=2, dim=0)
|
||||
|
||||
@@ -479,7 +479,7 @@ def rotmat2qvec(R):
|
||||
Rxx, Ryx, Rzx, Rxy, Ryy, Rzy, Rxz, Ryz, Rzz = R.flat
|
||||
K = (
|
||||
np.array(
|
||||
[
|
||||
[ # type: ignore
|
||||
[Rxx - Ryy - Rzz, 0, 0, 0],
|
||||
[Ryx + Rxy, Ryy - Rxx - Rzz, 0, 0],
|
||||
[Rzx + Rxz, Rzy + Ryz, Rzz - Rxx - Ryy, 0],
|
||||
|
||||
@@ -45,7 +45,7 @@ def get_semantics_and_mask_tensors_from_path(
|
||||
If no mask is required - use mask_indices = []
|
||||
"""
|
||||
if isinstance(mask_indices, List):
|
||||
mask_indices = torch.tensor(mask_indices, dtype="int64").view(1, 1, -1)
|
||||
mask_indices = torch.tensor(mask_indices, dtype=torch.int64).view(1, 1, -1)
|
||||
pil_image = Image.open(filepath)
|
||||
if scale_factor != 1.0:
|
||||
width, height = pil_image.size
|
||||
@@ -61,7 +61,7 @@ def get_depth_image_from_path(
|
||||
height: int,
|
||||
width: int,
|
||||
scale_factor: float,
|
||||
interpolation: int = cv2.INTER_NEAREST,
|
||||
interpolation: int = cv2.INTER_NEAREST, # type: ignore
|
||||
) -> torch.Tensor:
|
||||
"""Loads, rescales and resizes depth images.
|
||||
Filepath points to a 16-bit or 32-bit depth image, or a numpy array `*.npy`.
|
||||
@@ -78,9 +78,9 @@ def get_depth_image_from_path(
|
||||
"""
|
||||
if filepath.suffix == ".npy":
|
||||
image = np.load(filepath) * scale_factor
|
||||
image = cv2.resize(image, (width, height), interpolation=interpolation)
|
||||
image = cv2.resize(image, (width, height), interpolation=interpolation) # type: ignore
|
||||
else:
|
||||
image = cv2.imread(str(filepath.absolute()), cv2.IMREAD_ANYDEPTH)
|
||||
image = cv2.imread(str(filepath.absolute()), cv2.IMREAD_ANYDEPTH) # type: ignore
|
||||
image = image.astype(np.float64) * scale_factor
|
||||
image = cv2.resize(image, (width, height), interpolation=interpolation)
|
||||
image = cv2.resize(image, (width, height), interpolation=interpolation) # type: ignore
|
||||
return torch.from_numpy(image[:, :, np.newaxis])
|
||||
@@ -21,7 +21,7 @@ import concurrent.futures
|
||||
import multiprocessing
|
||||
import random
|
||||
from abc import abstractmethod
|
||||
from typing import Dict, Optional, Tuple, Union
|
||||
from typing import Dict, Optional, Sized, Tuple, Union
|
||||
|
||||
import torch
|
||||
from rich.progress import track
|
||||
@@ -58,6 +58,8 @@ class CacheDataloader(DataLoader):
|
||||
**kwargs,
|
||||
):
|
||||
self.dataset = dataset
|
||||
assert isinstance(self.dataset, Sized)
|
||||
|
||||
super().__init__(dataset=dataset, **kwargs) # This will set self.dataset
|
||||
self.num_times_to_repeat_images = num_times_to_repeat_images
|
||||
self.cache_all_images = (num_images_to_sample_from == -1) or (num_images_to_sample_from >= len(self.dataset))
|
||||
@@ -93,6 +95,7 @@ class CacheDataloader(DataLoader):
|
||||
def _get_batch_list(self):
|
||||
"""Returns a list of batches from the dataset attribute."""
|
||||
|
||||
assert isinstance(self.dataset, Sized)
|
||||
indices = random.sample(range(len(self.dataset)), k=self.num_images_to_sample_from)
|
||||
batch_list = []
|
||||
results = []
|
||||
@@ -183,6 +186,7 @@ class EvalDataloader(DataLoader):
|
||||
ray_bundle = self.cameras.generate_rays(camera_indices=image_idx, keep_shape=True)
|
||||
batch = self.input_dataset[image_idx]
|
||||
batch = get_dict_to_torch(batch, device=self.device, exclude=["image"])
|
||||
assert isinstance(batch, dict)
|
||||
return ray_bundle, batch
|
||||
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ Custom collate function that includes cases for nerfstudio types.
|
||||
import collections
|
||||
import collections.abc
|
||||
import re
|
||||
from typing import Callable, Dict, Union
|
||||
from typing import Any, Callable, Dict, Union
|
||||
|
||||
import torch
|
||||
import torch.utils.data
|
||||
@@ -33,9 +33,9 @@ NERFSTUDIO_COLLATE_ERR_MSG_FORMAT = (
|
||||
np_str_obj_array_pattern = re.compile(r"[SaUO]")
|
||||
|
||||
|
||||
def nerfstudio_collate(
|
||||
batch, extra_mappings: Union[Dict[type, Callable], None] = None
|
||||
): # pylint: disable=too-many-return-statements
|
||||
def nerfstudio_collate( # pylint: disable=too-many-return-statements
|
||||
batch: Any, extra_mappings: Union[Dict[type, Callable], None] = None
|
||||
) -> Any:
|
||||
r"""
|
||||
This is the default pytorch collate function, but with support for nerfstudio types. All documentation
|
||||
below is copied straight over from pytorch's default_collate function, python version 3.8.13,
|
||||
|
||||
Reference in new issue
Block a user