starting benchmarks

This commit is contained in:
Ethan Weber committed 2022-12-08 22:43:43 +00:00
1 parent 13fca7044c
commit 069e6f0f66
6 files changed
+102 -21

No files matched your search

+2
View File
@@ -228,6 +228,8 @@ class Config(PrintableConfig):
"""Which visualizer to use."""
data: Optional[Path] = None
"""Alias for --pipeline.datamanager.dataparser.data"""
wandb_name: Optional[str] = None
"""Alias for --logging.wandb.name"""
def is_viewer_enabled(self) -> bool:
"""Checks if a viewer is enabled."""
+25 -17
View File
@@ -99,6 +99,7 @@ class TCNNNerfactoField(Field):
hidden_dim_transient: int = 64,
appearance_embedding_dim: int = 32,
transient_embedding_dim: int = 16,
use_appearance_embedding: bool = True,
use_transient_embedding: bool = False,
use_semantics: bool = False,
num_semantic_classes: int = 100,
@@ -114,12 +115,15 @@ class TCNNNerfactoField(Field):
self.spatial_distortion = spatial_distortion
self.num_images = num_images
self.appearance_embedding_dim = appearance_embedding_dim
self.embedding_appearance = Embedding(self.num_images, self.appearance_embedding_dim)
self.use_average_appearance_embedding = use_average_appearance_embedding
self.use_appearance_embedding = use_appearance_embedding
self.use_transient_embedding = use_transient_embedding
self.use_semantics = use_semantics
self.use_pred_normals = use_pred_normals
if self.use_appearance_embedding:
self.embedding_appearance = Embedding(self.num_images, self.appearance_embedding_dim)
base_res = 16
features_per_level = 2
growth_factor = np.exp((np.log(max_res) - np.log(base_res)) / (num_levels - 1))
@@ -208,8 +212,9 @@ class TCNNNerfactoField(Field):
)
self.field_head_pred_normals = PredNormalsFieldHead(in_dim=self.mlp_pred_normals.n_output_dims)
appearance_embedding_dim = self.appearance_embedding_dim if self.use_appearance_embedding else 0
self.mlp_head = tcnn.Network(
n_input_dims=self.direction_encoding.n_output_dims + self.geo_feat_dim + self.appearance_embedding_dim,
n_input_dims=self.direction_encoding.n_output_dims + self.geo_feat_dim + appearance_embedding_dim,
n_output_dims=3,
network_config={
"otype": "FullyFusedMLP",
@@ -255,17 +260,18 @@ class TCNNNerfactoField(Field):
outputs_shape = ray_samples.frustums.directions.shape[:-1]
# appearance
if self.training:
embedded_appearance = self.embedding_appearance(camera_indices)
else:
if self.use_average_appearance_embedding:
embedded_appearance = torch.ones(
(*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device
) * self.embedding_appearance.mean(dim=0)
if self.use_appearance_embedding:
if self.training:
embedded_appearance = self.embedding_appearance(camera_indices)
else:
embedded_appearance = torch.zeros(
(*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device
)
if self.use_average_appearance_embedding:
embedded_appearance = torch.ones(
(*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device
) * self.embedding_appearance.mean(dim=0)
else:
embedded_appearance = torch.zeros(
(*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device
)
# transients
if self.use_transient_embedding and self.training:
@@ -304,12 +310,14 @@ class TCNNNerfactoField(Field):
x = self.mlp_pred_normals(pred_normals_inp).view(*outputs_shape, -1).to(directions)
outputs[FieldHeadNames.PRED_NORMALS] = self.field_head_pred_normals(x)
h_list = [
d,
density_embedding.view(-1, self.geo_feat_dim)
]
if self.use_appearance_embedding:
h_list += [embedded_appearance.view(-1, self.appearance_embedding_dim)]
h = torch.cat(
[
d,
density_embedding.view(-1, self.geo_feat_dim),
embedded_appearance.view(-1, self.appearance_embedding_dim),
],
h_list,
dim=-1,
)
rgb = self.mlp_head(h).view(*outputs_shape, -1).to(directions)
+4 -1
View File
@@ -71,7 +71,7 @@ class NerfactoModelConfig(ModelConfig):
"""Whether to randomize the background color."""
num_levels: int = 16
"""Number of levels of the hashmap for the base mlp."""
max_res: int = 1024
max_res: int = 2048
"""Maximum resolution of the hashmap for the base mlp."""
log2_hashmap_size: int = 19
"""Size of the hashmap for the base mlp"""
@@ -104,6 +104,8 @@ class NerfactoModelConfig(ModelConfig):
"""Predicted normal loss multiplier."""
use_proposal_weight_anneal: bool = True
"""Whether to use proposal weight annealing."""
use_appearance_embedding: bool = True
"""Whether to use appearance embedding."""
use_average_appearance_embedding: bool = True
"""Whether to use average appearance embedding or zeros for inference."""
proposal_weights_anneal_slope: float = 10.0
@@ -140,6 +142,7 @@ class NerfactoModel(Model):
spatial_distortion=scene_contraction,
num_images=self.num_train_data,
use_pred_normals=self.config.predict_normals,
use_appearance_embedding=self.config.use_appearance_embedding,
use_average_appearance_embedding=self.config.use_average_appearance_embedding,
)
+5 -2
View File
@@ -198,7 +198,7 @@ def setup_event_writer(config: cfg.Config, log_dir: Path) -> None:
"""
using_event_writer = False
if config.is_wandb_enabled():
curr_writer = WandbWriter(log_dir=log_dir)
curr_writer = WandbWriter(log_dir=log_dir, wandb_name=config.wandb_name)
EVENT_WRITERS.append(curr_writer)
using_event_writer = True
if config.is_tensorboard_enabled():
@@ -282,8 +282,11 @@ class TimeWriter:
class WandbWriter(Writer):
"""WandDB Writer Class"""
def __init__(self, log_dir: Path):
def __init__(self, log_dir: Path, wandb_name: Optional[str] = None):
wandb.init(project="nerfstudio-project", dir=str(log_dir), reinit=True)
if wandb_name:
wandb.run.name = wandb_name
wandb.run.save()
def write_image(self, name: str, image: TensorType["H", "W", "C"], step: int) -> None:
image = torch.permute(image, (2, 0, 1))
+66
View File
@@ -0,0 +1,66 @@
"""
Process the Mipnerf360 dataset.
"""
from nerfstudio.utils.scripts import run_command
# # split = ["bicycle", "flowers", "garden", "stump", "treehill", "room", "counter", "kitchen", "bonsai"]
# splits = ["bicycle", "garden", "stump", "room", "counter", "kitchen", "bonsai"]
# # for processing data
# for split in splits:
# print(f"Processing {split}...")
# working_dir = "/home/ethanweber/nerfactory/data/_prep/images/mipnerf360"
# cmd = f"ns-process-data images --data {working_dir}/{split}/images --output-dir {working_dir}/processed/{split}/"
# output = run_command(cmd, verbose=True)
# for training
# 1/8 of input images used in the paper = 0.125 -> 1 - this = 0.875
# ns-train nerfacto --vis wandb --data data/nerfstudio/garden nerfstudio-data --downscale-factor 1 --train-split-percentage 0.875
# X turn off pose optimization
# turn off appearance optimization
# increase hash map size. max_res 2048
# ns-train nerfacto --vis wandb --data data/_prep/images/mipnerf360/processed/garden --pipeline.datamanager.camera-optimizer.mode off --trainer.steps-per-eval-all-images 5000 nerfstudio-data --downscale-factor 4
# mipnerf 360 experiments
splits = ["bicycle", "garden", "stump", "room", "counter", "kitchen", "bonsai"] # 7 splits
gpus = [0, 1, 3, 4, 5, 6, 7]
commands = []
for split, gpu in zip(splits, gpus):
wandb_name = f"mipnerf360-{split}"
command = " ".join(
(
f"CUDA_VISIBLE_DEVICES={gpu}",
"ns-train nerfacto",
"--vis wandb",
f"--wandb-name {wandb_name}",
f"--data data/_prep/images/mipnerf360/processed/{split}",
"--pipeline.datamanager.camera-optimizer.mode off",
"--trainer.steps-per-eval-all-images 5000",
"--trainer.max-num-iterations 250000",
"--pipeline.model.use_appearance_embedding False",
"nerfstudio-data",
"--downscale-factor 4",
)
)
print(command)
commands.append(command)
# import subprocess
# import sys
# def f(cmd):
# out = subprocess.run(cmd, capture_output=False, shell=True, check=False)
# if out.returncode != 0:
# print(f"Error running command: {cmd}")
# sys.exit(1)
# if out.stdout is not None:
# return out.stdout.decode("utf-8")
# from multiprocessing import Pool
# with Pool() as p:
# print(p.map(f, commands))
-1
View File
@@ -235,7 +235,6 @@ class PhototourismDownload(DatasetDownload):
shutil.rmtree(tmp_path)
os.remove(download_path)
Commands = Union[
Annotated[BlenderDownload, tyro.conf.subcommand(name="blender")],
Annotated[FriendsDownload, tyro.conf.subcommand(name="friends")],