Monodepth Regularization (#2284)

* monodepth implemented

* device

* pair pixel sampler

* docstring

* removing unused imports

* fix bugs

* cache in cache dir, no longer edit json file

* moving to depth nerfacto

* fixed bug using median depth by accident

* removing depthdataset from nerfacto

* added pixelsamplerconfig

* added equirectangular warning back

* bug fix

* delete sample_config.py + small fixes

* rename to pixel_sampler

* cleaning

* test

* passing num_rays_per_batch properly

* fixed transforms typo

* renaming in nerfacto

* remove depth files from test data

* removed defaulting to base pixel sampler with mask

* adding patch_size back

---------

Co-authored-by: AdamRashid96 <abrashid@berkeley.edu>
Co-authored-by: Matthew Tancik <tancik@berkeley.edu>
This commit is contained in:
authored and GitHub committed 2023-08-01 18:52:47 -07:00
1 parent 72da747fc2
commit 71c6b9cf3d
8 files changed
+293 -89

No files matched your search

+79 -7
View File
@@ -18,31 +18,95 @@ Depth dataset.
from typing import Dict
import numpy as np
from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs
from nerfstudio.model_components import losses
from nerfstudio.data.datasets.base_dataset import InputDataset
from nerfstudio.data.utils.data_utils import get_depth_image_from_path
from nerfstudio.utils.rich_utils import CONSOLE
from typing import Union
from PIL import Image
import torch
from rich.progress import track
from pathlib import Path
import json
class DepthDataset(InputDataset):
"""Dataset that returns images and depths.
"""Dataset that returns images and depths. If no depths are found, then we generate them with Zoe Depth.
Args:
dataparser_outputs: description of where and how to read input images.
scale_factor: The scaling factor for the dataparser outputs.
"""
exclude_batch_keys_from_device = InputDataset.exclude_batch_keys_from_device + ["depth_image"]
def __init__(self, dataparser_outputs: DataparserOutputs, scale_factor: float = 1.0):
super().__init__(dataparser_outputs, scale_factor)
assert (
"depth_filenames" in dataparser_outputs.metadata.keys()
and dataparser_outputs.metadata["depth_filenames"] is not None
)
# if there are no depth images than we want to generate them all with zoe depth
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if len(dataparser_outputs.image_filenames) > 0 and (
"depth_filenames" not in dataparser_outputs.metadata.keys()
or dataparser_outputs.metadata["depth_filenames"] is None
):
losses.DEPTH_METRIC = 0
CONSOLE.print("[bold yellow] No depth data found!")
cache = dataparser_outputs.image_filenames[0].parent / "depths.npy"
if cache.exists():
CONSOLE.print("[bold yellow] Loading cache!")
# load all the depths
self.depths = np.load(cache)
self.depths = torch.from_numpy(self.depths).to(device)
else:
depth_tensors = []
transforms = self._find_transform(dataparser_outputs.image_filenames[0])
data = dataparser_outputs.image_filenames[0].parent
if transforms is not None:
meta = json.load(open(transforms, "r"))
frames = meta["frames"]
filenames = [data / frames[j]["file_path"].split("/")[-1] for j in range(len(frames))]
else:
meta = None
frames = None
filenames = dataparser_outputs.image_filenames
repo = "isl-org/ZoeDepth"
self.zoe = torch.compile(torch.hub.load(repo, "ZoeD_NK", pretrained=True).to(device))
for i in track(range(len(filenames)), description="Generating depth images"):
image_filename = filenames[i]
pil_image = Image.open(image_filename)
image = np.array(pil_image, dtype="uint8") # shape is (h, w) or (h, w, 3 or 4)
if len(image.shape) == 2:
image = image[:, :, None].repeat(3, axis=2)
image = torch.from_numpy(image.astype("float32") / 255.0)
with torch.no_grad():
image = torch.permute(image, (2, 0, 1)).unsqueeze(0).to(device)
if image.shape[1] == 4:
image = image[:, :3, :, :]
depth_tensor = self.zoe.infer(image).squeeze().unsqueeze(-1)
depth_tensors.append(depth_tensor)
self.depths = torch.stack(depth_tensors)
np.save(cache, self.depths.cpu().numpy())
dataparser_outputs.metadata["depth_filenames"] = None
dataparser_outputs.metadata["depth_unit_scale_factor"] = 1.0
self.metadata["depth_filenames"] = None
self.metadata["depth_unit_scale_factor"] = 1.0
else:
losses.DEPTH_METRIC = 0
self.depth_filenames = self.metadata["depth_filenames"]
self.depth_unit_scale_factor = self.metadata["depth_unit_scale_factor"]
def get_metadata(self, data: Dict) -> Dict:
if self.depth_filenames is None:
return {"depth_image": self.depths[data["image_idx"]]}
filepath = self.depth_filenames[data["image_idx"]]
height = int(self._dataparser_outputs.cameras.height[data["image_idx"]])
width = int(self._dataparser_outputs.cameras.width[data["image_idx"]])
@@ -54,3 +118,11 @@ class DepthDataset(InputDataset):
)
return {"depth_image": depth_image}
def _find_transform(self, image_path: Path) -> Union[Path, None]:
while image_path.parent != image_path:
transform_path = image_path.parent / "transforms.json"
if transform_path.exists():
return transform_path
image_path = image_path.parent
return None