More typing fixes (#1941)

* Fix more types

* Fix tests

* Minor cleaning

* Address comments

* Resolve linting errors

* Re-ignore pymeshlab type

* Add extra assert / error message to mipnerf model
This commit is contained in:
Brent Yi authored and GitHub committed 2023-05-17 07:17:40 -07:00
1 parent 8b9574bb5a
commit 814db32ec4
53 files changed
+386 -205

No files matched your search

@@ -22,7 +22,18 @@ import functools
from abc import abstractmethod
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, Generic, List, Literal, Optional, Tuple, Type, Union
from typing import (
Any,
Callable,
Dict,
Generic,
List,
Literal,
Optional,
Tuple,
Type,
Union,
)
import torch
from torch import nn
@@ -72,7 +83,7 @@ def variable_res_collate(batch: List[Dict]) -> Dict:
if mask is not None:
masks.append(mask)
new_batch: dict = nerfstudio_collate(batch)
new_batch = nerfstudio_collate(batch)
new_batch["image"] = images
if masks:
new_batch["mask"] = masks
@@ -314,7 +325,7 @@ class VanillaDataManagerConfig(DataManagerConfig):
camera_optimizer: CameraOptimizerConfig = CameraOptimizerConfig()
"""Specifies the camera pose optimizer used during training. Helpful if poses are noisy, such as for data from
Record3D."""
collate_fn = staticmethod(nerfstudio_collate)
collate_fn: Callable[[Any], Any] = staticmethod(nerfstudio_collate)
"""Specifies the collate function to use for the train and eval dataloaders."""
camera_res_scale_factor: float = 1.0
"""The scale factor for scaling spatial data such as images, mask, semantics
@@ -356,7 +367,7 @@ class VanillaDataManager(DataManager, Generic[TDataset]): # pylint: disable=abs
local_rank: int = 0,
**kwargs, # pylint: disable=unused-argument
):
self.dataset_type: Type[TDataset] = kwargs.get("_dataset_type", TDataset.__default__)
self.dataset_type: Type[TDataset] = kwargs.get("_dataset_type", getattr(TDataset, "__default__"))
self.config = config
self.device = device
self.world_size = world_size
@@ -486,6 +497,7 @@ class VanillaDataManager(DataManager, Generic[TDataset]): # pylint: disable=abs
self.train_count += 1
image_batch = next(self.iter_train_image_dataloader)
assert self.train_pixel_sampler is not None
assert isinstance(image_batch, dict)
batch = self.train_pixel_sampler.sample(image_batch)
ray_indices = batch["indices"]
ray_bundle = self.train_ray_generator(ray_indices)
@@ -496,6 +508,7 @@ class VanillaDataManager(DataManager, Generic[TDataset]): # pylint: disable=abs
self.eval_count += 1
image_batch = next(self.iter_eval_image_dataloader)
assert self.eval_pixel_sampler is not None
assert isinstance(image_batch, dict)
batch = self.eval_pixel_sampler.sample(image_batch)
ray_indices = batch["indices"]
ray_bundle = self.eval_ray_generator(ray_indices)
@@ -50,7 +50,7 @@ def traj_string_to_matrix(traj_string: str):
ts = tokens[0]
# Rotation in angle axis
angle_axis = [float(tokens[1]), float(tokens[2]), float(tokens[3])]
r_w_to_p, _ = cv2.Rodrigues(np.asarray(angle_axis))
r_w_to_p, _ = cv2.Rodrigues(np.asarray(angle_axis)) # type: ignore
# Translation
t_w_to_p = np.asarray([float(tokens[4]), float(tokens[5]), float(tokens[6])])
extrinsics = np.eye(4, 4)
@@ -213,6 +213,7 @@ class ARKitScenes(DataParser):
@staticmethod
def _get_pose(frame_id: str, poses_from_traj: dict):
frame_pose = None
if str(frame_id) in poses_from_traj:
frame_pose = np.array(poses_from_traj[str(frame_id)])
else:
@@ -220,6 +221,7 @@ class ARKitScenes(DataParser):
if abs(float(frame_id) - float(my_key)) < 0.005:
frame_pose = np.array(poses_from_traj[str(my_key)])
assert frame_pose is not None
frame_pose[0:3, 1:3] *= -1
frame_pose = frame_pose[np.array([1, 0, 2, 3]), :]
frame_pose[2, :] *= -1
@@ -52,7 +52,7 @@ def downscale(img, scale: int) -> np.ndarray:
if height % scale > 0 or width % scale > 0:
raise ValueError(f"Image shape ({height},{width}) must be divisible by the" f" scale ({scale}).")
out_height, out_width = height // scale, width // scale
resized = cv2.resize(img, (out_width, out_height), cv2.INTER_AREA)
resized = cv2.resize(img, (out_width, out_height), cv2.INTER_AREA) # type: ignore
return resized
@@ -70,11 +70,11 @@ def upscale(img, scale: int) -> np.ndarray:
return img
height, width = img.shape[:2]
out_height, out_width = height * scale, width * scale
resized = cv2.resize(img, (out_width, out_height), cv2.INTER_AREA)
resized = cv2.resize(img, (out_width, out_height), cv2.INTER_AREA) # type: ignore
return resized
def rescale(img, scale_factor: float, interpolation=cv2.INTER_AREA) -> np.ndarray:
def rescale(img, scale_factor: float, interpolation=cv2.INTER_AREA) -> np.ndarray: # type: ignore
"""Function from DyCheck's repo. Rescale an image.
Args:
@@ -106,7 +106,7 @@ def rescale(img, scale_factor: float, interpolation=cv2.INTER_AREA) -> np.ndarra
out_width = math.ceil(width * scale_factor)
out_width -= out_width % 2
return cv2.resize(img, (out_width, out_height), interpolation)
return cv2.resize(img, (out_width, out_height), interpolation) # type: ignore
def _load_scene_info(data_dir: Path) -> Tuple[np.ndarray, float, float, float]:
@@ -242,7 +242,7 @@ class Dycheck(DataParser):
frame_names = np.array(split_dict["frame_names"])[[0]]
time_ids = np.array(split_dict["time_ids"])[[0]]
image_filenames, depth_filenames, cams = self.process_frames(frame_names, time_ids)
image_filenames, depth_filenames, cams = self.process_frames(frame_names.tolist(), time_ids)
scene_box = SceneBox(
aabb=torch.tensor(
@@ -316,13 +316,13 @@ class Dycheck(DataParser):
d = self.config.downscale_factor
if not image_filenames[0].exists():
CONSOLE.print(f"downscale factor {d}x not exist, converting")
ori_h, ori_w = cv2.imread(str(self.data / f"rgb/1x/{frame_names[0]}.png")).shape[:2]
ori_h, ori_w = cv2.imread(str(self.data / f"rgb/1x/{frame_names[0]}.png")).shape[:2] # type: ignore
(self.data / f"rgb/{d}x").mkdir(exist_ok=True)
h, w = ori_h // d, ori_w // d
for frame in frame_names:
cv2.imwrite(
cv2.imwrite( # type: ignore
str(self.data / f"rgb/{d}x/{frame}.png"),
cv2.resize(cv2.imread(str(self.data / f"rgb/1x/{frame}.png")), [h, w]),
cv2.resize(cv2.imread(str(self.data / f"rgb/1x/{frame}.png")), [h, w]), # type: ignore
)
CONSOLE.print("finished")
@@ -331,8 +331,8 @@ class Dycheck(DataParser):
(self.data / f"processed_depth/{d}x").mkdir(exist_ok=True, parents=True)
for idx, frame in enumerate(frame_names):
depth = np.load(self.data / f"depth/1x/{frame}.npy")
mask = rescale((depth != 0).astype(np.uint8) * 255, 1 / d, cv2.INTER_AREA)
depth = rescale(depth, 1 / d, cv2.INTER_AREA)
mask = rescale((depth != 0).astype(np.uint8) * 255, 1 / d, cv2.INTER_AREA) # type: ignore
depth = rescale(depth, 1 / d, cv2.INTER_AREA) # type: ignore
depth[mask != 255] = 0
depth = _rescale_depth(depth, cams[idx])
np.save(str(self.data / f"processed_depth/{d}x/{frame}.npy"), depth)
@@ -57,7 +57,6 @@ class MinimalDataParser(DataParser):
if "mask_filenames" in data.keys():
mask_filenames = [filepath.parent / path for path in data["mask_filenames"].tolist()]
metadata = None
if "semantics" in data.keys():
semantics = data["semantics"].item()
metadata = {
@@ -68,6 +67,8 @@ class MinimalDataParser(DataParser):
mask_classes=semantics["mask_classes"].tolist(),
)
}
else:
metadata = {}
scene_box_aabb = torch.from_numpy(data["scene_box"])
scene_box = SceneBox(aabb=scene_box_aabb)
@@ -90,7 +91,7 @@ class MinimalDataParser(DataParser):
)
applied_scale = 1.0
applied_transform = np.eye(4, dtype=np.float32)[:3, :]
applied_transform = torch.eye(4, dtype=torch.float32)[:3, :]
if "applied_scale" in data.keys():
applied_scale = float(data["applied_scale"])
if "applied_transform" in data.keys():
@@ -87,7 +87,11 @@ class NuScenes(DataParser):
def _generate_dataparser_outputs(self, split="train"):
# pylint: disable=too-many-statements
nusc = NuScenesDatabase(version=self.config.version, dataroot=self.config.data_dir, verbose=self.config.verbose)
nusc = NuScenesDatabase(
version=self.config.version,
dataroot=str(self.config.data_dir.absolute()),
verbose=self.config.verbose,
)
cameras = ["CAM_" + camera for camera in self.config.cameras]
assert (
@@ -81,7 +81,7 @@ class ScanNet(DataParser):
depth_dir_sorted = list(sorted(depth_dir.iterdir(), key=lambda x: int(x.name.split(".")[0])))
pose_dir_sorted = list(sorted(pose_dir.iterdir(), key=lambda x: int(x.name.split(".")[0])))
first_img = cv2.imread(img_dir_sorted[0].as_posix())
first_img = cv2.imread(str(img_dir_sorted[0].absolute())) # type: ignore
h, w, _ = first_img.shape
image_filenames, depth_filenames, intrinsics, poses = [], [], [], []
@@ -111,11 +111,12 @@ class Sitcoms3D(DataParser):
lengths = scene_box.aabb[1] - scene_box.aabb[0]
longest_dim = torch.argmax(lengths)
longest_length = lengths[longest_dim]
scale = scene_scale / longest_length
scale = scene_scale / longest_length.item()
scene_box.aabb = scene_box.aabb * scale # box
camera_to_worlds[..., 3] *= scale # cameras
# --- semantics ---
semantics = None
if self.config.include_semantics:
empty_path = Path()
replace_this_path = str(empty_path / images_folder / empty_path)
+2 -4
View File
@@ -44,7 +44,6 @@ class InputDataset(Dataset):
def __init__(self, dataparser_outputs: DataparserOutputs, scale_factor: float = 1.0):
super().__init__()
self._dataparser_outputs = dataparser_outputs
self.has_masks = dataparser_outputs.mask_filenames is not None
self.scale_factor = scale_factor
self.scene_box = deepcopy(dataparser_outputs.scene_box)
self.metadata = deepcopy(dataparser_outputs.metadata)
@@ -95,9 +94,8 @@ class InputDataset(Dataset):
image_idx: The image index in the dataset.
"""
image = self.get_image(image_idx)
data = {"image_idx": image_idx}
data["image"] = image
if self.has_masks:
data = {"image_idx": image_idx, "image": image}
if self._dataparser_outputs.mask_filenames is not None:
mask_filepath = self._dataparser_outputs.mask_filenames[image_idx]
data["mask"] = get_image_mask_tensor_from_path(filepath=mask_filepath, scale_factor=self.scale_factor)
assert (
+1 -1
View File
@@ -81,7 +81,7 @@ class SDFDataset(InputDataset):
normal = normal * 2.0 - 1.0 # omnidata output is normalized so we convert it back to normal here
normal = torch.from_numpy(normal).float()
rot = camtoworld[:3, :3]
rot = torch.from_numpy(camtoworld[:3, :3])
normal_map = normal.reshape(3, -1)
normal_map = torch.nn.functional.normalize(normal_map, p=2, dim=0)
@@ -479,7 +479,7 @@ def rotmat2qvec(R):
Rxx, Ryx, Rzx, Rxy, Ryy, Rzy, Rxz, Ryz, Rzz = R.flat
K = (
np.array(
[
[ # type: ignore
[Rxx - Ryy - Rzz, 0, 0, 0],
[Ryx + Rxy, Ryy - Rxx - Rzz, 0, 0],
[Rzx + Rxz, Rzy + Ryz, Rzz - Rxx - Ryy, 0],
+5 -5
View File
@@ -45,7 +45,7 @@ def get_semantics_and_mask_tensors_from_path(
If no mask is required - use mask_indices = []
"""
if isinstance(mask_indices, List):
mask_indices = torch.tensor(mask_indices, dtype="int64").view(1, 1, -1)
mask_indices = torch.tensor(mask_indices, dtype=torch.int64).view(1, 1, -1)
pil_image = Image.open(filepath)
if scale_factor != 1.0:
width, height = pil_image.size
@@ -61,7 +61,7 @@ def get_depth_image_from_path(
height: int,
width: int,
scale_factor: float,
interpolation: int = cv2.INTER_NEAREST,
interpolation: int = cv2.INTER_NEAREST, # type: ignore
) -> torch.Tensor:
"""Loads, rescales and resizes depth images.
Filepath points to a 16-bit or 32-bit depth image, or a numpy array `*.npy`.
@@ -78,9 +78,9 @@ def get_depth_image_from_path(
"""
if filepath.suffix == ".npy":
image = np.load(filepath) * scale_factor
image = cv2.resize(image, (width, height), interpolation=interpolation)
image = cv2.resize(image, (width, height), interpolation=interpolation) # type: ignore
else:
image = cv2.imread(str(filepath.absolute()), cv2.IMREAD_ANYDEPTH)
image = cv2.imread(str(filepath.absolute()), cv2.IMREAD_ANYDEPTH) # type: ignore
image = image.astype(np.float64) * scale_factor
image = cv2.resize(image, (width, height), interpolation=interpolation)
image = cv2.resize(image, (width, height), interpolation=interpolation) # type: ignore
return torch.from_numpy(image[:, :, np.newaxis])
+5 -1
View File
@@ -21,7 +21,7 @@ import concurrent.futures
import multiprocessing
import random
from abc import abstractmethod
from typing import Dict, Optional, Tuple, Union
from typing import Dict, Optional, Sized, Tuple, Union
import torch
from rich.progress import track
@@ -58,6 +58,8 @@ class CacheDataloader(DataLoader):
**kwargs,
):
self.dataset = dataset
assert isinstance(self.dataset, Sized)
super().__init__(dataset=dataset, **kwargs) # This will set self.dataset
self.num_times_to_repeat_images = num_times_to_repeat_images
self.cache_all_images = (num_images_to_sample_from == -1) or (num_images_to_sample_from >= len(self.dataset))
@@ -93,6 +95,7 @@ class CacheDataloader(DataLoader):
def _get_batch_list(self):
"""Returns a list of batches from the dataset attribute."""
assert isinstance(self.dataset, Sized)
indices = random.sample(range(len(self.dataset)), k=self.num_images_to_sample_from)
batch_list = []
results = []
@@ -183,6 +186,7 @@ class EvalDataloader(DataLoader):
ray_bundle = self.cameras.generate_rays(camera_indices=image_idx, keep_shape=True)
batch = self.input_dataset[image_idx]
batch = get_dict_to_torch(batch, device=self.device, exclude=["image"])
assert isinstance(batch, dict)
return ray_bundle, batch
+4 -4
View File
@@ -19,7 +19,7 @@ Custom collate function that includes cases for nerfstudio types.
import collections
import collections.abc
import re
from typing import Callable, Dict, Union
from typing import Any, Callable, Dict, Union
import torch
import torch.utils.data
@@ -33,9 +33,9 @@ NERFSTUDIO_COLLATE_ERR_MSG_FORMAT = (
np_str_obj_array_pattern = re.compile(r"[SaUO]")
def nerfstudio_collate(
batch, extra_mappings: Union[Dict[type, Callable], None] = None
): # pylint: disable=too-many-return-statements
def nerfstudio_collate( # pylint: disable=too-many-return-statements
batch: Any, extra_mappings: Union[Dict[type, Callable], None] = None
) -> Any:
r"""
This is the default pytorch collate function, but with support for nerfstudio types. All documentation
below is copied straight over from pytorch's default_collate function, python version 3.8.13,