mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-16 04:43:52 +08:00
* knocking out a couple * Fix more errors * Add pyright * Remove some unnecessary cv2 type ignores, warning on missing imports (for missing tinycudann) * Minor annotation fixes * Update docs * Drop `None` from temporal distortion output --------- Co-authored-by: Brent Yi <yibrenth@gmail.com>
342 lines
13 KiB
Python
342 lines
13 KiB
Python
# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Data parser for DyCheck (https://arxiv.org/abs/2210.13445) dataset of `iphone` subset"""
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Dict, List, Tuple, Type
|
|
|
|
import cv2
|
|
import numpy as np
|
|
import torch
|
|
|
|
from nerfstudio.cameras.cameras import Cameras, CameraType
|
|
from nerfstudio.data.dataparsers.base_dataparser import (
|
|
DataParser,
|
|
DataParserConfig,
|
|
DataparserOutputs,
|
|
)
|
|
from nerfstudio.data.scene_box import SceneBox
|
|
from nerfstudio.utils.colors import get_color
|
|
from nerfstudio.utils.io import load_from_json
|
|
from nerfstudio.utils.rich_utils import CONSOLE
|
|
|
|
|
|
def downscale(img, scale: int) -> np.ndarray:
|
|
"""Function from DyCheck's repo. Downscale an image.
|
|
|
|
Args:
|
|
img: Input image
|
|
scale: Factor of the scale
|
|
|
|
Returns:
|
|
New image
|
|
"""
|
|
if scale == 1:
|
|
return img
|
|
height, width = img.shape[:2]
|
|
if height % scale > 0 or width % scale > 0:
|
|
raise ValueError(f"Image shape ({height},{width}) must be divisible by the" f" scale ({scale}).")
|
|
out_height, out_width = height // scale, width // scale
|
|
resized = cv2.resize(img, (out_width, out_height), cv2.INTER_AREA) # type: ignore
|
|
return resized
|
|
|
|
|
|
def upscale(img, scale: int) -> np.ndarray:
|
|
"""Function from DyCheck's repo. Upscale an image.
|
|
|
|
Args:
|
|
img: Input image
|
|
scale: Factor of the scale
|
|
|
|
Returns:
|
|
New image
|
|
"""
|
|
if scale == 1:
|
|
return img
|
|
height, width = img.shape[:2]
|
|
out_height, out_width = height * scale, width * scale
|
|
resized = cv2.resize(img, (out_width, out_height), cv2.INTER_AREA) # type: ignore
|
|
return resized
|
|
|
|
|
|
def rescale(img, scale_factor: float, interpolation: int = cv2.INTER_AREA) -> np.ndarray:
|
|
"""Function from DyCheck's repo. Rescale an image.
|
|
|
|
Args:
|
|
img: Input image
|
|
scale: Factor of the scale
|
|
interpolation: Interpolation method in opencv
|
|
|
|
Returns:
|
|
New image
|
|
"""
|
|
scale_factor = float(scale_factor)
|
|
if scale_factor <= 0.0:
|
|
raise ValueError("scale_factor must be a non-negative number.")
|
|
if scale_factor == 1.0:
|
|
return img
|
|
|
|
height, width = img.shape[:2]
|
|
if scale_factor.is_integer():
|
|
return upscale(img, int(scale_factor))
|
|
|
|
inv_scale = 1.0 / scale_factor
|
|
if inv_scale.is_integer() and (scale_factor * height).is_integer() and (scale_factor * width).is_integer():
|
|
return downscale(img, int(inv_scale))
|
|
|
|
print(f"Resizing image by non-integer factor {scale_factor}, this may lead to artifacts.")
|
|
height, width = img.shape[:2]
|
|
out_height = math.ceil(height * scale_factor)
|
|
out_height -= out_height % 2
|
|
out_width = math.ceil(width * scale_factor)
|
|
out_width -= out_width % 2
|
|
|
|
return cv2.resize(img, (out_width, out_height), interpolation) # type: ignore
|
|
|
|
|
|
def _load_scene_info(data_dir: Path) -> Tuple[np.ndarray, float, float, float]:
|
|
"""Function from DyCheck's repo. Load scene info from json.
|
|
|
|
Args:
|
|
data_dir: data path
|
|
|
|
Returns:
|
|
A tuple of scene info: center, scale, near, far
|
|
"""
|
|
scene_dict = load_from_json(data_dir / "scene.json")
|
|
center = np.array(scene_dict["center"], dtype=np.float32)
|
|
scale = scene_dict["scale"]
|
|
near = scene_dict["near"]
|
|
far = scene_dict["far"]
|
|
return center, scale, near, far
|
|
|
|
|
|
def _load_metadata_info(data_dir: Path) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
|
|
"""Function from DyCheck's repo. Load scene metadata from json.
|
|
|
|
Args:
|
|
data_dir: data path
|
|
|
|
Returns:
|
|
A tuple of scene info: frame_names_map, time_ids, camera_ids
|
|
"""
|
|
dataset_dict = load_from_json(data_dir / "dataset.json")
|
|
_frame_names = np.array(dataset_dict["ids"])
|
|
|
|
metadata_dict = load_from_json(data_dir / "metadata.json")
|
|
time_ids = np.array([metadata_dict[k]["warp_id"] for k in _frame_names], dtype=np.uint32)
|
|
camera_ids = np.array([metadata_dict[k]["camera_id"] for k in _frame_names], dtype=np.uint32)
|
|
|
|
frame_names_map = np.zeros((time_ids.max() + 1, camera_ids.max() + 1), _frame_names.dtype)
|
|
for i, (t, c) in enumerate(zip(time_ids, camera_ids)):
|
|
frame_names_map[t, c] = _frame_names[i]
|
|
|
|
return frame_names_map, time_ids, camera_ids
|
|
|
|
|
|
def _rescale_depth(depth_raw: np.ndarray, cam: Dict) -> np.ndarray:
|
|
"""Depth rescale function from DyCheck.
|
|
|
|
Args:
|
|
depth: A numpy ndarray of the raw depth
|
|
cam: Dict of the camera
|
|
|
|
Returns:
|
|
A numpy ndarray of the processed depth
|
|
"""
|
|
xx, yy = np.meshgrid(np.arange(cam["width"], dtype=np.float32), np.arange(cam["height"], dtype=np.float32))
|
|
pixels = np.stack([xx, yy], axis=-1)
|
|
batch_shape = pixels.shape[:-1]
|
|
pixels = np.reshape(pixels, (-1, 2))
|
|
y = (pixels[..., 1] - cam["cy"]) / cam["fy"]
|
|
x = (pixels[..., 0] - cam["cx"]) / cam["fx"]
|
|
# x = (pixels[..., 0] - self.principal_point_x - y * self.skew) / self.scale_factor_x
|
|
# assume skew = 0
|
|
viewdirs = np.stack([x, y, np.ones_like(x)], axis=-1)
|
|
local_viewdirs = viewdirs / np.linalg.norm(viewdirs, axis=-1, keepdims=True)
|
|
viewdirs = (cam["camera_to_worlds"][:3, :3] @ local_viewdirs[..., None])[..., 0]
|
|
viewdirs /= np.linalg.norm(viewdirs, axis=-1, keepdims=True)
|
|
viewdirs = viewdirs.reshape((*batch_shape, 3))
|
|
cosa = viewdirs @ (cam["camera_to_worlds"][:, 2])
|
|
depth = depth_raw / cosa[..., None]
|
|
return depth
|
|
|
|
|
|
@dataclass
|
|
class DycheckDataParserConfig(DataParserConfig):
|
|
"""Dycheck (https://arxiv.org/abs/2210.13445) dataset parser config"""
|
|
|
|
_target: Type = field(default_factory=lambda: Dycheck)
|
|
"""target class to instantiate"""
|
|
data: Path = Path("data/iphone/mochi-high-five")
|
|
"""Directory specifying location of data."""
|
|
scale_factor: float = 5.0
|
|
"""How much to scale the camera origins by."""
|
|
alpha_color: str = "white"
|
|
"""alpha color of background"""
|
|
downscale_factor: int = 1
|
|
"""How much to downscale images."""
|
|
scene_box_bound: float = 1.5
|
|
"""Boundary of scene box."""
|
|
|
|
|
|
@dataclass
|
|
class Dycheck(DataParser):
|
|
"""Dycheck (https://arxiv.org/abs/2210.13445) Dataset `iphone` subset"""
|
|
|
|
config: DycheckDataParserConfig
|
|
includes_time: bool = True
|
|
|
|
def __init__(self, config: DycheckDataParserConfig):
|
|
super().__init__(config=config)
|
|
self.data: Path = config.data
|
|
self.scale_factor: float = config.scale_factor
|
|
self.alpha_color = config.alpha_color
|
|
# load extra info from "extra.json"
|
|
extra_path = self.data / "extra.json"
|
|
extra_dict = load_from_json(extra_path)
|
|
self._factor = extra_dict["factor"]
|
|
self._fps = extra_dict["fps"]
|
|
self._bbox = np.array(extra_dict["bbox"], dtype=np.float32)
|
|
self._lookat = np.array(extra_dict["lookat"], dtype=np.float32)
|
|
self._up = np.array(extra_dict["up"], dtype=np.float32)
|
|
self._center, self._scale, self._near, self._far = _load_scene_info(self.data)
|
|
self._frame_names_map, self._time_ids, self._camera_ids = _load_metadata_info(self.data)
|
|
|
|
def _generate_dataparser_outputs(self, split="train"):
|
|
if self.alpha_color is not None:
|
|
alpha_color_tensor = get_color(self.alpha_color)
|
|
else:
|
|
alpha_color_tensor = None
|
|
splits_dir = self.data / "splits"
|
|
|
|
# scale the scene to fill the aabb bbox
|
|
sf = self.config.scene_box_bound / 4 / (self._scale * self._far)
|
|
# CONSOLE.print(f"scale factor changed from {self.config.scale_factor} to {sf}")
|
|
self.config.scale_factor = sf
|
|
|
|
if not (splits_dir / f"{split}.json").exists():
|
|
CONSOLE.print(f"split {split} not found, using split train")
|
|
split = "train"
|
|
split_dict = load_from_json(splits_dir / f"{split}.json")
|
|
frame_names = np.array(split_dict["frame_names"])
|
|
time_ids = np.array(split_dict["time_ids"])
|
|
if split != "train":
|
|
CONSOLE.print(f"split {split} is empty, using the 1st training image")
|
|
split_dict = load_from_json(splits_dir / "train.json")
|
|
frame_names = np.array(split_dict["frame_names"])[[0]]
|
|
time_ids = np.array(split_dict["time_ids"])[[0]]
|
|
|
|
image_filenames, depth_filenames, cams = self.process_frames(frame_names.tolist(), time_ids)
|
|
|
|
scene_box = SceneBox(
|
|
aabb=torch.tensor(
|
|
[[-self.config.scene_box_bound] * 3, [self.config.scene_box_bound] * 3], dtype=torch.float32
|
|
)
|
|
)
|
|
cam_dict = {}
|
|
for k in cams[0].keys():
|
|
cam_dict[k] = torch.stack([torch.as_tensor(c[k]) for c in cams], dim=0)
|
|
cameras = Cameras(camera_type=CameraType.PERSPECTIVE, **cam_dict)
|
|
|
|
scale = self._scale * self.config.scale_factor
|
|
dataparser_outputs = DataparserOutputs(
|
|
image_filenames=image_filenames,
|
|
cameras=cameras,
|
|
alpha_color=alpha_color_tensor,
|
|
scene_box=scene_box,
|
|
metadata={
|
|
"depth_filenames": depth_filenames,
|
|
"depth_unit_scale_factor": scale,
|
|
"scale": scale,
|
|
"near": self._near * scale,
|
|
"far": self._far * scale,
|
|
},
|
|
)
|
|
|
|
return dataparser_outputs
|
|
|
|
def process_frames(self, frame_names: List[str], time_ids: np.ndarray) -> Tuple[List, List, List]:
|
|
"""Read cameras and filenames from the name list.
|
|
|
|
Args:
|
|
frame_names: list of file names.
|
|
time_ids: time id of each frame.
|
|
|
|
Returns:
|
|
A list of camera, each entry is a dict of the camera.
|
|
"""
|
|
image_filenames, depth_filenames = [], []
|
|
cams = []
|
|
for idx, frame in enumerate(frame_names):
|
|
image_filenames.append(self.data / f"rgb/{self.config.downscale_factor}x/{frame}.png")
|
|
depth_filenames.append(self.data / f"processed_depth/{self.config.downscale_factor}x/{frame}.npy")
|
|
cam_json = load_from_json(self.data / f"camera/{frame}.json")
|
|
c2w = torch.as_tensor(cam_json["orientation"]).T
|
|
position = torch.as_tensor(cam_json["position"])
|
|
position -= self._center # some scenes look weird (wheel)
|
|
position *= self._scale * self.config.scale_factor
|
|
pose = torch.zeros([3, 4])
|
|
pose[:3, :3] = c2w
|
|
pose[:3, 3] = position
|
|
# from opencv coord to opengl coord (used by nerfstudio)
|
|
pose[0:3, 1:3] *= -1 # switch cam coord x,y
|
|
pose = pose[[1, 0, 2], :] # switch world x,y
|
|
pose[2, :] *= -1 # invert world z
|
|
# for aabb bbox usage
|
|
pose = pose[[1, 2, 0], :] # switch world xyz to zxy
|
|
cams.append(
|
|
{
|
|
"camera_to_worlds": pose,
|
|
"fx": cam_json["focal_length"] / self.config.downscale_factor,
|
|
"fy": cam_json["focal_length"] * cam_json["pixel_aspect_ratio"] / self.config.downscale_factor,
|
|
"cx": cam_json["principal_point"][0] / self.config.downscale_factor,
|
|
"cy": cam_json["principal_point"][1] / self.config.downscale_factor,
|
|
"height": cam_json["image_size"][1] // self.config.downscale_factor,
|
|
"width": cam_json["image_size"][0] // self.config.downscale_factor,
|
|
"times": torch.as_tensor(time_ids[idx] / self._time_ids.max()).float(),
|
|
}
|
|
)
|
|
|
|
d = self.config.downscale_factor
|
|
if not image_filenames[0].exists():
|
|
CONSOLE.print(f"downscale factor {d}x not exist, converting")
|
|
ori_h, ori_w = cv2.imread(str(self.data / f"rgb/1x/{frame_names[0]}.png")).shape[:2]
|
|
(self.data / f"rgb/{d}x").mkdir(exist_ok=True)
|
|
h, w = ori_h // d, ori_w // d
|
|
for frame in frame_names:
|
|
cv2.imwrite(
|
|
str(self.data / f"rgb/{d}x/{frame}.png"),
|
|
cv2.resize(cv2.imread(str(self.data / f"rgb/1x/{frame}.png")), (h, w)),
|
|
)
|
|
CONSOLE.print("finished")
|
|
|
|
if not depth_filenames[0].exists():
|
|
CONSOLE.print(f"processed depth downscale factor {d}x not exist, converting")
|
|
(self.data / f"processed_depth/{d}x").mkdir(exist_ok=True, parents=True)
|
|
for idx, frame in enumerate(frame_names):
|
|
depth = np.load(self.data / f"depth/1x/{frame}.npy")
|
|
mask = rescale((depth != 0).astype(np.uint8) * 255, 1 / d, cv2.INTER_AREA)
|
|
depth = rescale(depth, 1 / d, cv2.INTER_AREA)
|
|
depth[mask != 255] = 0
|
|
depth = _rescale_depth(depth, cams[idx])
|
|
np.save(str(self.data / f"processed_depth/{d}x/{frame}.npy"), depth)
|
|
CONSOLE.print("finished")
|
|
|
|
return image_filenames, depth_filenames, cams
|