Train and eval data splits (Nerfbusters) (#2207)

* nerfbusters dataparser

* nerfbuster dataparser and visualization of eval views

* resolved some errors from  pyright

* enable ns-process-data to take two videos as input and use one for eval and the other for train

* resolve some pyright issues

* instead of data being a tuple, I added eval_data as another argument

* implemented train eval split for all cases

* implemented train eval split for all cases

* reverting equi changes

* reverting reality capture changes

* raise valueerror

* revert comment

* added split override code back to dataparser

* reverted one change

* using image border now

* adding docs

---------

Co-authored-by: Ethan Weber <eweb0124@gmail.com>
This commit is contained in:
Frederik WarburgandEthan Weber authored and GitHub committed 2023-08-04 10:11:55 -04:00
1 parent 388fcb09e1
commit 9b03299d40
14 files changed
+288 -83

No files matched your search

@@ -15,7 +15,6 @@
from __future__ import annotations
import math
from dataclasses import dataclass, field
from pathlib import Path
from typing import Literal, Optional, Type
@@ -26,12 +25,14 @@ from PIL import Image
from nerfstudio.cameras import camera_utils
from nerfstudio.cameras.cameras import CAMERA_MODEL_TO_TYPE, Cameras, CameraType
from nerfstudio.data.dataparsers.base_dataparser import (
DataParser,
DataParserConfig,
DataparserOutputs,
)
from nerfstudio.data.dataparsers.base_dataparser import DataParser, DataParserConfig, DataparserOutputs
from nerfstudio.data.scene_box import SceneBox
from nerfstudio.data.utils.dataparsers_utils import (
get_train_eval_split_filename,
get_train_eval_split_fraction,
get_train_eval_split_interval,
get_train_eval_split_all,
)
from nerfstudio.utils.io import load_from_json
from nerfstudio.utils.rich_utils import CONSOLE
@@ -58,8 +59,18 @@ class NerfstudioDataParserConfig(DataParserConfig):
"""The method to use to center the poses."""
auto_scale_poses: bool = True
"""Whether to automatically scale the poses to fit in +/- 1 bounding box."""
eval_mode: Literal["fraction", "filename", "interval", "all"] = "fraction"
"""
The method to use for splitting the dataset into train and eval.
Fraction splits based on a percentage for train and the remaining for eval.
Filename splits based on filenames containing train/eval.
Interval uses every nth frame for eval.
All uses all the images for any split.
"""
train_split_fraction: float = 0.9
"""The fraction of images to use for training. The remaining images are for eval."""
"""The percentage of the dataset to use for training. Only used when eval_mode is train-split-fraction."""
eval_interval: int = 8
"""The interval between frames to use for eval. Only used when eval_mode is eval-interval."""
depth_unit_scale_factor: float = 1e-3
"""Scales the depth values to meters. Default value is 0.001 for a millimeter to meter conversion."""
@@ -105,9 +116,18 @@ class Nerfstudio(DataParser):
width = []
distort = []
# sort the frames by fname
fnames = []
for frame in meta["frames"]:
filepath = Path(frame["file_path"])
fname = self._get_fname(filepath, data_dir)
fnames.append(fname)
inds = np.argsort(fnames)
frames = [meta["frames"][ind] for ind in inds]
for frame in frames:
filepath = Path(frame["file_path"])
fname = self._get_fname(filepath, data_dir)
if not fx_fixed:
assert "fl_x" in frame, "fx not specified in frame"
@@ -182,16 +202,21 @@ class Nerfstudio(DataParser):
elif has_split_files_spec:
raise RuntimeError(f"The dataset's list of filenames for split {split} is missing.")
else:
# filter image_filenames and poses based on train/eval split percentage
num_images = len(image_filenames)
num_train_images = math.ceil(num_images * self.config.train_split_fraction)
num_eval_images = num_images - num_train_images
i_all = np.arange(num_images)
i_train = np.linspace(
0, num_images - 1, num_train_images, dtype=int
) # equally spaced training images starting and ending at 0 and num_images-1
i_eval = np.setdiff1d(i_all, i_train) # eval images are the remaining images
assert len(i_eval) == num_eval_images
# find train and eval indices based on the eval_mode specified
if self.config.eval_mode == "fraction":
i_train, i_eval = get_train_eval_split_fraction(image_filenames, self.config.train_split_fraction)
elif self.config.eval_mode == "filename":
i_train, i_eval = get_train_eval_split_filename(image_filenames)
elif self.config.eval_mode == "interval":
i_train, i_eval = get_train_eval_split_interval(image_filenames, self.config.eval_interval)
elif self.config.eval_mode == "all":
CONSOLE.log(
"[yellow] Be careful with '--eval-mode=all'. If using camera optimization, the cameras may diverge in the current implementation, giving unpredictable results."
)
i_train, i_eval = get_train_eval_split_all(image_filenames)
else:
raise ValueError(f"Unknown eval mode {self.config.eval_mode}")
if split == "train":
indices = i_train
elif split in ["val", "test"]:
+101
View File
@@ -0,0 +1,101 @@
# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Data parser utils for nerfstudio datasets. """
import math
import os
from typing import List, Tuple
import numpy as np
def get_train_eval_split_fraction(image_filenames: List, train_split_fraction: float) -> Tuple[np.ndarray, np.ndarray]:
"""
Get the train/eval split fraction based on the number of images and the train split fraction.
Args:
image_filenames: list of image filenames
train_split_fraction: fraction of images to use for training
"""
# filter image_filenames and poses based on train/eval split percentage
num_images = len(image_filenames)
num_train_images = math.ceil(num_images * train_split_fraction)
num_eval_images = num_images - num_train_images
i_all = np.arange(num_images)
i_train = np.linspace(
0, num_images - 1, num_train_images, dtype=int
) # equally spaced training images starting and ending at 0 and num_images-1
i_eval = np.setdiff1d(i_all, i_train) # eval images are the remaining images
assert len(i_eval) == num_eval_images
return i_train, i_eval
def get_train_eval_split_filename(image_filenames: List) -> Tuple[np.ndarray, np.ndarray]:
"""
Get the train/eval split based on the filename of the images.
Args:
image_filenames: list of image filenames
"""
num_images = len(image_filenames)
basenames = [os.path.basename(image_filename) for image_filename in image_filenames]
i_all = np.arange(num_images)
i_train = []
i_eval = []
for idx, basename in zip(i_all, basenames):
# check the frame index
if "train" in basename:
i_train.append(idx)
elif "eval" in basename:
i_eval.append(idx)
else:
raise ValueError("frame should contain train/eval in its name to use this eval-frame-index eval mode")
return np.array(i_train), np.array(i_eval)
def get_train_eval_split_interval(image_filenames: List, eval_interval: float) -> Tuple[np.ndarray, np.ndarray]:
"""
Get the train/eval split based on the interval of the images.
Args:
image_filenames: list of image filenames
eval_interval: interval of images to use for eval
"""
num_images = len(image_filenames)
all_indices = np.arange(num_images)
train_indices = all_indices[all_indices % eval_interval != 0]
eval_indices = all_indices[all_indices % eval_interval == 0]
i_train = train_indices
i_eval = eval_indices
return i_train, i_eval
def get_train_eval_split_all(image_filenames: List) -> Tuple[np.ndarray, np.ndarray]:
"""
Get the train/eval split where all indices are used for both train and eval.
Args:
image_filenames: list of image filenames
"""
num_images = len(image_filenames)
i_all = np.arange(num_images)
i_train = i_all
i_eval = i_all
return i_train, i_eval