Added cache capabilities for data

This commit is contained in:
Chung Min Kim committed 2023-04-02 23:19:24 +00:00
1 parent 5952a692eb
commit d8b2462e50
7 files changed
+70 -43

No files matched your search

+7 -17
View File
@@ -85,17 +85,18 @@ class LERFDataManager(VanillaDataManager): # pylint: disable=abstract-method
images = torch.cat(images)
cache_dir = f"outputs/{self.config.dataparser.data.name}"
# clip_cache_path, dino_cache_path = self._search_cache(cache_dir)
clip_cache_path = osp.join(cache_dir, "clip")
dino_cache_path = osp.join(cache_dir, "dino.npy")
clip_cache_path = Path(osp.join(cache_dir, f"clip_{self.image_encoder.name}"))
dino_cache_path = Path(osp.join(cache_dir, "dino.npy"))
# NOTE: cache config is sensitive to list vs. tuple, because it checks for dict equality
self.clip_interpolator = PyramidEmbeddingDataloader(
image_list=images,
device=self.device,
cfg={
"tile_size_range": (0.05, 0.5),
"tile_size_range": [0.05, 0.5],
"tile_size_res": 7,
"stride_scaler": 0.5,
"image_shape": tuple(images.shape[2:4]),
"image_shape": list(images.shape[2:4]),
"model_name": self.image_encoder.name
},
cache_path=clip_cache_path,
model=self.image_encoder,
@@ -103,21 +104,10 @@ class LERFDataManager(VanillaDataManager): # pylint: disable=abstract-method
self.dino_dataloader = DinoDataloader(
image_list=images,
device=self.device,
cfg={"image_shape": tuple(images.shape[2:4])},
cfg={"image_shape": list(images.shape[2:4])},
cache_path=dino_cache_path,
)
# def _search_cache(self, cache_path, prop_dict):
# if osp.exists(cache_path):
# # load yaml file
# with open(cache_path, "r") as f:
# cache = yaml.load(f, Loader=yaml.FullLoader)
# for k, v in cache.items():
# if k in self.config:
# self.config[k] = v
# else:
# ...
def next_train(self, step: int) -> Tuple[RayBundle, Dict]:
"""Returns the next batch of data from the train dataloader."""
self.train_count += 1
@@ -2,6 +2,7 @@ import os
import typing
from abc import ABC, ABCMeta, abstractmethod
from pathlib import Path
import json
import numpy as np
import torch
@@ -30,14 +31,24 @@ class FeatureDataloader(ABC):
pass
def load(self):
cache_info_path = self.cache_path.with_suffix(".info")
if not cache_info_path.exists():
raise FileNotFoundError
with open(cache_info_path, "r") as f:
cfg = json.loads(f.read())
if cfg != self.cfg:
raise ValueError("Config mismatch")
self.data = torch.from_numpy(np.load(self.cache_path))
def save(self):
cache_info_path = self.cache_path.with_suffix(".info")
with open(cache_info_path, "w") as f:
f.write(json.dumps(self.cfg))
np.save(self.cache_path, self.data)
def try_load(self, img_list: torch.Tensor):
try:
self.load()
except FileNotFoundError:
except (FileNotFoundError, ValueError):
self.create(img_list)
self.save()
@@ -17,6 +17,7 @@ class PatchEmbeddingDataloader(FeatureDataloader):
assert "tile_ratio" in cfg
assert "stride_ratio" in cfg
assert "image_shape" in cfg
assert "model_name" in cfg
self.kernel_size = int(cfg["image_shape"][0] * cfg["tile_ratio"])
self.stride = int(self.kernel_size * cfg["stride_ratio"])
@@ -6,6 +6,8 @@ from lerf.data.utils.feature_dataloader import FeatureDataloader
from lerf.data.utils.patch_embedding_dataloader import PatchEmbeddingDataloader
from lerf.encoders.image_encoder import BaseImageEncoder
from tqdm import tqdm
from pathlib import Path
import json
class PyramidEmbeddingDataloader(FeatureDataloader):
@@ -21,9 +23,10 @@ class PyramidEmbeddingDataloader(FeatureDataloader):
assert "tile_size_res" in cfg
assert "stride_scaler" in cfg
assert "image_shape" in cfg
assert "model_name" in cfg
cfg["tile_sizes"] = torch.linspace(*cfg["tile_size_range"], cfg["tile_size_res"]).to(device)
cfg["strider_scaler_list"] = [self._stride_scaler(tr.item(), cfg["stride_scaler"]) for tr in cfg["tile_sizes"]]
self.tile_sizes = torch.linspace(*cfg["tile_size_range"], cfg["tile_size_res"]).to(device)
self.strider_scaler_list = [self._stride_scaler(tr.item(), cfg["stride_scaler"]) for tr in self.tile_sizes]
self.model = model
self.embed_size = self.model.embedding_dim
@@ -41,38 +44,58 @@ class PyramidEmbeddingDataloader(FeatureDataloader):
def load(self):
# don't create anything, PatchEmbeddingDataloader will create itself
cache_info_path = self.cache_path.with_suffix(".info")
# check if cache exists
if not cache_info_path.exists():
raise FileNotFoundError
# if config is different, remove all cached content
with open(cache_info_path, "r") as f:
cfg = json.loads(f.read())
if cfg != self.cfg:
for f in os.listdir(self.cache_path):
os.remove(os.path.join(self.cache_path, f))
raise ValueError("Config mismatch")
raise FileNotFoundError # trigger create
def create(self, image_list):
os.makedirs(self.cache_path, exist_ok=True)
for i, tr in enumerate(tqdm(self.cfg["tile_sizes"], desc="Scales")):
stride_scaler = self.cfg["strider_scaler_list"][i]
for i, tr in enumerate(tqdm(self.tile_sizes, desc="Scales")):
stride_scaler = self.strider_scaler_list[i]
self.data_dict[i] = PatchEmbeddingDataloader(
cfg={"tile_ratio": tr.item(), "stride_ratio": stride_scaler, "image_shape": self.cfg["image_shape"]},
cfg={
"tile_ratio": tr.item(),
"stride_ratio": stride_scaler,
"image_shape": self.cfg["image_shape"],
"model_name": self.cfg["model_name"]
},
device=self.device,
model=self.model,
image_list=image_list,
cache_path=f"{self.cache_path}/level_{i}.npy",
cache_path=Path(f"{self.cache_path}/level_{i}.npy"),
)
def save(self):
cache_info_path = self.cache_path.with_suffix(".info")
with open(cache_info_path, "w") as f:
f.write(json.dumps(self.cfg))
# don't save anything, PatchEmbeddingDataloader will save itself
pass
def _random_scales(self, img_points):
# img_points: (B, 3) # (img_ind, x, y)
# return: (B, 512), some random scale (between 0, 1)
tile_sizes = self.cfg["tile_sizes"]
random_scale_bin = torch.randint(tile_sizes.shape[0] - 1, size=(img_points.shape[0],), device=self.device)
random_scale_bin = torch.randint(self.tile_sizes.shape[0] - 1, size=(img_points.shape[0],), device=self.device)
random_scale_weight = torch.rand(img_points.shape[0], dtype=torch.float16, device=self.device)
stepsize = (tile_sizes[1] - tile_sizes[0]) / (tile_sizes[-1] - tile_sizes[0])
stepsize = (self.tile_sizes[1] - self.tile_sizes[0]) / (self.tile_sizes[-1] - self.tile_sizes[0])
bottom_interp = torch.zeros((img_points.shape[0], self.embed_size), dtype=torch.float16, device=self.device)
top_interp = torch.zeros((img_points.shape[0], self.embed_size), dtype=torch.float16, device=self.device)
for i in range(len(tile_sizes) - 1):
for i in range(len(self.tile_sizes) - 1):
bottom_interp[random_scale_bin == i] = self.data_dict[i](img_points[random_scale_bin == i])
top_interp[random_scale_bin == i] = self.data_dict[i + 1](img_points[random_scale_bin == i])
@@ -82,12 +105,10 @@ class PyramidEmbeddingDataloader(FeatureDataloader):
)
def _uniform_scales(self, img_points, scale):
tile_sizes = self.cfg["tile_sizes"]
scale_bin = torch.floor(
(scale - tile_sizes[0]) / (tile_sizes[-1] - tile_sizes[0]) * (tile_sizes.shape[0] - 1)
(scale - self.tile_sizes[0]) / (self.tile_sizes[-1] - self.tile_sizes[0]) * (self.tile_sizes.shape[0] - 1)
).to(torch.int64)
scale_weight = (scale - tile_sizes[scale_bin]) / (tile_sizes[scale_bin + 1] - tile_sizes[scale_bin])
scale_weight = (scale - self.tile_sizes[scale_bin]) / (self.tile_sizes[scale_bin + 1] - self.tile_sizes[scale_bin])
interp_lst = torch.stack([interp(img_points) for interp in self.data_dict.values()])
point_inds = torch.arange(img_points.shape[0])
interp = torch.lerp(
+4 -3
View File
@@ -14,9 +14,6 @@ from lerf.encoders.image_encoder import BaseImageEncoder, BaseImageEncoderConfig
@dataclass
class CLIPNetworkConfig(BaseImageEncoderConfig):
_target: Type = field(default_factory=lambda: CLIPNetwork)
name: str = "clip"
description: str = "OpenAI CLIP network, images and text"
"""target class to instantiate"""
clip_model_type: str = "ViT-B-16"
clip_n_dims: int = 512
negatives: Tuple[str] = ("object", "things", "stuff", "texture")
@@ -56,6 +53,10 @@ class CLIPNetwork(BaseImageEncoder):
self.pos_embeds.shape[1] == self.clip_n_dims
), "Embedding dimensionality must match the model dimensionality"
@property
def name(self) -> str:
return "clip_{}".format(self.config.clip_model_type)
@property
def embedding_dim(self) -> int:
return self.config.clip_n_dims
+6 -4
View File
@@ -9,12 +9,14 @@ from nerfstudio.configs import base_config as cfg
@dataclass
class BaseImageEncoderConfig(cfg.InstantiateConfig):
_target: Type = field(default_factory=lambda: BaseImageEncoder)
name: str = ""
"""name of the encoder"""
description: str = ""
"""description of the encoder"""
class BaseImageEncoder:
@abstractproperty
def name(self) -> str:
"""
returns the name of the encoder
"""
@abstractproperty
def embedding_dim(self) -> int:
"""
@@ -15,9 +15,6 @@ from lerf.encoders.image_encoder import BaseImageEncoder, BaseImageEncoderConfig
@dataclass
class OpenCLIPNetworkConfig(BaseImageEncoderConfig):
_target: Type = field(default_factory=lambda: OpenCLIPNetwork)
name: str = "openclip_vit_b_16"
description: str = "OpenCLIP network, images and text"
"""target class to instantiate"""
clip_model_type: str = "ViT-B-16"
clip_model_pretrained: str = "laion2b_s34b_b88k"
clip_n_dims: int = 512
@@ -62,6 +59,10 @@ class OpenCLIPNetwork(BaseImageEncoder):
self.pos_embeds.shape[1] == self.clip_n_dims
), "Embedding dimensionality must match the model dimensionality"
@property
def name(self) -> str:
return "openclip_{}_{}".format(self.config.clip_model_type, self.config.clip_model_pretrained)
@property
def embedding_dim(self) -> int:
return self.config.clip_n_dims