Add mipnerf 360 data support

This commit is contained in:
Matt Tancik authored and Matthew Tancik committed 2022-06-28 10:51:35 -07:00
1 parent 73a2449bae
commit f71c2c21f2
2 files changed
+129

No files matched your search

+122
View File
@@ -0,0 +1,122 @@
# Copyright 2022 The Plenoptix Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Code for loading mipnerf 360 dataset formats.
"""
import os
import imageio
import numpy as np
import torch
from pyrad.data.structs import DatasetInputs, SceneBounds
def load_mipnerf_360_data(
basedir: str,
downscale_factor: int = 1,
split: str = "train",
val_skip: int = 8,
auto_scale: bool = True,
aabb_scale: float = 1.0,
) -> DatasetInputs:
"""Processes mipnerf 360 data.
Example data can be downloaded from https://jonbarron.info/mipnerf360/.
Args:
data_directory (str): Location of data
downscale_factor (int, optional): How much to downscale images. Must be a factor of 2. Defaults to 1.
split (str, optional): Which dataset split to generate.
val_skip (int, optional): 1/val_skip images to use for validation. Defaults to 8.
auto_scale (boolean, optional): Scale based on pose bounds. Defaults to True.
aabb_scale (float, optional): Scene scale, Defaults to 1.0.
Returns:
DatasetInputs
"""
image_dir = os.path.join(basedir, "images")
if downscale_factor > 1:
image_dir += f"_{downscale_factor}"
if not os.path.exists(image_dir):
raise ValueError(f"Image directory {image_dir} doesn't exist")
valid_formats = [".jpg", ".png"]
image_filenames = []
for f in os.listdir(image_dir):
ext = os.path.splitext(f)[1]
if ext.lower() not in valid_formats:
continue
image_filenames.append(os.path.join(image_dir, f))
image_filenames = sorted(image_filenames)
num_images = len(image_filenames)
poses_bounds = np.load(os.path.join(basedir, "poses_bounds.npy"))
poses = poses_bounds[:, :-2].reshape([-1, 3, 5]).astype(np.float32)
bounds = poses_bounds[:, -2:].transpose([1, 0])
# Scale factor used in mipnerf
if auto_scale:
scale_factor = 1 / (np.min(poses_bounds) * 0.75)
poses[:, :3, 3] *= scale_factor
bounds *= scale_factor
# Center poses
center = poses[:, :3, 3].mean(0)
poses[:, :3, :3] -= center
if num_images != poses.shape[0]:
raise RuntimeError(f"Different number of images ({num_images}), and poses ({poses.shape[0]})")
idx_test = np.arange(num_images)[::val_skip]
idx_train = np.array([i for i in np.arange(num_images) if i not in idx_test])
idx = idx_train if split == "train" else idx_test
image_filenames = np.array(image_filenames)[idx]
poses = poses[idx]
img_0 = imageio.imread(image_filenames[0])
image_height, image_width = img_0.shape[:2]
poses[:, :2, 4] = np.array([image_height, image_width])
poses[:, 2, 4] = poses[:, 2, 4] * 1.0 / downscale_factor
focal_length = poses[0, -1, -1]
cx = image_width / 2.0
cy = image_height / 2.0
camera_to_world = torch.from_numpy(poses[:, :3, :4]) # camera to world transform
num_cameras = len(image_filenames)
num_intrinsics_params = 3
intrinsics = torch.ones((num_cameras, num_intrinsics_params), dtype=torch.float32)
intrinsics *= torch.tensor([cx, cy, focal_length])
scene_bounds = SceneBounds(
aabb=torch.tensor(
[[-aabb_scale, -aabb_scale, -aabb_scale], [aabb_scale, aabb_scale, aabb_scale]], dtype=torch.float32
)
)
dataset_inputs = DatasetInputs(
image_filenames=image_filenames,
downscale_factor=1,
intrinsics=intrinsics,
camera_to_world=camera_to_world,
scene_bounds=scene_bounds,
)
return dataset_inputs
+7
View File
@@ -24,6 +24,7 @@ from omegaconf import ListConfig
from pyrad.data.format.blender import load_blender_data
from pyrad.data.format.friends import load_friends_data
from pyrad.data.format.instant_ngp import load_instant_ngp_data
from pyrad.data.format.mipnerf_360 import load_mipnerf_360_data
from pyrad.data.structs import DatasetInputs
from pyrad.utils.colors import get_color
from pyrad.utils.io import get_absolute_path, load_from_pkl, make_dir, write_to_pkl
@@ -96,6 +97,12 @@ def get_dataset_inputs(
downscale_factor=downscale_factor,
split="train",
)
elif dataset_format == "mipnerf_360":
dataset_inputs = load_mipnerf_360_data(
get_absolute_path(data_directory),
downscale_factor=downscale_factor,
split="train",
)
else:
raise NotImplementedError(f"{dataset_format} is not a valid dataset type")