mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-17 21:33:56 +08:00
starting benchmarks
This commit is contained in:
1 parent
13fca7044c
commit
069e6f0f66
6 files changed
+102
-21
No files matched your search
@@ -228,6 +228,8 @@ class Config(PrintableConfig):
|
||||
"""Which visualizer to use."""
|
||||
data: Optional[Path] = None
|
||||
"""Alias for --pipeline.datamanager.dataparser.data"""
|
||||
wandb_name: Optional[str] = None
|
||||
"""Alias for --logging.wandb.name"""
|
||||
|
||||
def is_viewer_enabled(self) -> bool:
|
||||
"""Checks if a viewer is enabled."""
|
||||
|
||||
@@ -99,6 +99,7 @@ class TCNNNerfactoField(Field):
|
||||
hidden_dim_transient: int = 64,
|
||||
appearance_embedding_dim: int = 32,
|
||||
transient_embedding_dim: int = 16,
|
||||
use_appearance_embedding: bool = True,
|
||||
use_transient_embedding: bool = False,
|
||||
use_semantics: bool = False,
|
||||
num_semantic_classes: int = 100,
|
||||
@@ -114,12 +115,15 @@ class TCNNNerfactoField(Field):
|
||||
self.spatial_distortion = spatial_distortion
|
||||
self.num_images = num_images
|
||||
self.appearance_embedding_dim = appearance_embedding_dim
|
||||
self.embedding_appearance = Embedding(self.num_images, self.appearance_embedding_dim)
|
||||
self.use_average_appearance_embedding = use_average_appearance_embedding
|
||||
self.use_appearance_embedding = use_appearance_embedding
|
||||
self.use_transient_embedding = use_transient_embedding
|
||||
self.use_semantics = use_semantics
|
||||
self.use_pred_normals = use_pred_normals
|
||||
|
||||
if self.use_appearance_embedding:
|
||||
self.embedding_appearance = Embedding(self.num_images, self.appearance_embedding_dim)
|
||||
|
||||
base_res = 16
|
||||
features_per_level = 2
|
||||
growth_factor = np.exp((np.log(max_res) - np.log(base_res)) / (num_levels - 1))
|
||||
@@ -208,8 +212,9 @@ class TCNNNerfactoField(Field):
|
||||
)
|
||||
self.field_head_pred_normals = PredNormalsFieldHead(in_dim=self.mlp_pred_normals.n_output_dims)
|
||||
|
||||
appearance_embedding_dim = self.appearance_embedding_dim if self.use_appearance_embedding else 0
|
||||
self.mlp_head = tcnn.Network(
|
||||
n_input_dims=self.direction_encoding.n_output_dims + self.geo_feat_dim + self.appearance_embedding_dim,
|
||||
n_input_dims=self.direction_encoding.n_output_dims + self.geo_feat_dim + appearance_embedding_dim,
|
||||
n_output_dims=3,
|
||||
network_config={
|
||||
"otype": "FullyFusedMLP",
|
||||
@@ -255,17 +260,18 @@ class TCNNNerfactoField(Field):
|
||||
outputs_shape = ray_samples.frustums.directions.shape[:-1]
|
||||
|
||||
# appearance
|
||||
if self.training:
|
||||
embedded_appearance = self.embedding_appearance(camera_indices)
|
||||
else:
|
||||
if self.use_average_appearance_embedding:
|
||||
embedded_appearance = torch.ones(
|
||||
(*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device
|
||||
) * self.embedding_appearance.mean(dim=0)
|
||||
if self.use_appearance_embedding:
|
||||
if self.training:
|
||||
embedded_appearance = self.embedding_appearance(camera_indices)
|
||||
else:
|
||||
embedded_appearance = torch.zeros(
|
||||
(*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device
|
||||
)
|
||||
if self.use_average_appearance_embedding:
|
||||
embedded_appearance = torch.ones(
|
||||
(*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device
|
||||
) * self.embedding_appearance.mean(dim=0)
|
||||
else:
|
||||
embedded_appearance = torch.zeros(
|
||||
(*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device
|
||||
)
|
||||
|
||||
# transients
|
||||
if self.use_transient_embedding and self.training:
|
||||
@@ -304,12 +310,14 @@ class TCNNNerfactoField(Field):
|
||||
x = self.mlp_pred_normals(pred_normals_inp).view(*outputs_shape, -1).to(directions)
|
||||
outputs[FieldHeadNames.PRED_NORMALS] = self.field_head_pred_normals(x)
|
||||
|
||||
h_list = [
|
||||
d,
|
||||
density_embedding.view(-1, self.geo_feat_dim)
|
||||
]
|
||||
if self.use_appearance_embedding:
|
||||
h_list += [embedded_appearance.view(-1, self.appearance_embedding_dim)]
|
||||
h = torch.cat(
|
||||
[
|
||||
d,
|
||||
density_embedding.view(-1, self.geo_feat_dim),
|
||||
embedded_appearance.view(-1, self.appearance_embedding_dim),
|
||||
],
|
||||
h_list,
|
||||
dim=-1,
|
||||
)
|
||||
rgb = self.mlp_head(h).view(*outputs_shape, -1).to(directions)
|
||||
|
||||
@@ -71,7 +71,7 @@ class NerfactoModelConfig(ModelConfig):
|
||||
"""Whether to randomize the background color."""
|
||||
num_levels: int = 16
|
||||
"""Number of levels of the hashmap for the base mlp."""
|
||||
max_res: int = 1024
|
||||
max_res: int = 2048
|
||||
"""Maximum resolution of the hashmap for the base mlp."""
|
||||
log2_hashmap_size: int = 19
|
||||
"""Size of the hashmap for the base mlp"""
|
||||
@@ -104,6 +104,8 @@ class NerfactoModelConfig(ModelConfig):
|
||||
"""Predicted normal loss multiplier."""
|
||||
use_proposal_weight_anneal: bool = True
|
||||
"""Whether to use proposal weight annealing."""
|
||||
use_appearance_embedding: bool = True
|
||||
"""Whether to use appearance embedding."""
|
||||
use_average_appearance_embedding: bool = True
|
||||
"""Whether to use average appearance embedding or zeros for inference."""
|
||||
proposal_weights_anneal_slope: float = 10.0
|
||||
@@ -140,6 +142,7 @@ class NerfactoModel(Model):
|
||||
spatial_distortion=scene_contraction,
|
||||
num_images=self.num_train_data,
|
||||
use_pred_normals=self.config.predict_normals,
|
||||
use_appearance_embedding=self.config.use_appearance_embedding,
|
||||
use_average_appearance_embedding=self.config.use_average_appearance_embedding,
|
||||
)
|
||||
|
||||
|
||||
@@ -198,7 +198,7 @@ def setup_event_writer(config: cfg.Config, log_dir: Path) -> None:
|
||||
"""
|
||||
using_event_writer = False
|
||||
if config.is_wandb_enabled():
|
||||
curr_writer = WandbWriter(log_dir=log_dir)
|
||||
curr_writer = WandbWriter(log_dir=log_dir, wandb_name=config.wandb_name)
|
||||
EVENT_WRITERS.append(curr_writer)
|
||||
using_event_writer = True
|
||||
if config.is_tensorboard_enabled():
|
||||
@@ -282,8 +282,11 @@ class TimeWriter:
|
||||
class WandbWriter(Writer):
|
||||
"""WandDB Writer Class"""
|
||||
|
||||
def __init__(self, log_dir: Path):
|
||||
def __init__(self, log_dir: Path, wandb_name: Optional[str] = None):
|
||||
wandb.init(project="nerfstudio-project", dir=str(log_dir), reinit=True)
|
||||
if wandb_name:
|
||||
wandb.run.name = wandb_name
|
||||
wandb.run.save()
|
||||
|
||||
def write_image(self, name: str, image: TensorType["H", "W", "C"], step: int) -> None:
|
||||
image = torch.permute(image, (2, 0, 1))
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
"""
|
||||
Process the Mipnerf360 dataset.
|
||||
"""
|
||||
|
||||
from nerfstudio.utils.scripts import run_command
|
||||
|
||||
# # split = ["bicycle", "flowers", "garden", "stump", "treehill", "room", "counter", "kitchen", "bonsai"]
|
||||
# splits = ["bicycle", "garden", "stump", "room", "counter", "kitchen", "bonsai"]
|
||||
|
||||
|
||||
# # for processing data
|
||||
|
||||
# for split in splits:
|
||||
# print(f"Processing {split}...")
|
||||
# working_dir = "/home/ethanweber/nerfactory/data/_prep/images/mipnerf360"
|
||||
# cmd = f"ns-process-data images --data {working_dir}/{split}/images --output-dir {working_dir}/processed/{split}/"
|
||||
# output = run_command(cmd, verbose=True)
|
||||
|
||||
# for training
|
||||
|
||||
# 1/8 of input images used in the paper = 0.125 -> 1 - this = 0.875
|
||||
# ns-train nerfacto --vis wandb --data data/nerfstudio/garden nerfstudio-data --downscale-factor 1 --train-split-percentage 0.875
|
||||
# X turn off pose optimization
|
||||
# turn off appearance optimization
|
||||
# increase hash map size. max_res 2048
|
||||
# ns-train nerfacto --vis wandb --data data/_prep/images/mipnerf360/processed/garden --pipeline.datamanager.camera-optimizer.mode off --trainer.steps-per-eval-all-images 5000 nerfstudio-data --downscale-factor 4
|
||||
|
||||
|
||||
# mipnerf 360 experiments
|
||||
|
||||
splits = ["bicycle", "garden", "stump", "room", "counter", "kitchen", "bonsai"] # 7 splits
|
||||
gpus = [0, 1, 3, 4, 5, 6, 7]
|
||||
|
||||
commands = []
|
||||
for split, gpu in zip(splits, gpus):
|
||||
wandb_name = f"mipnerf360-{split}"
|
||||
command = " ".join(
|
||||
(
|
||||
f"CUDA_VISIBLE_DEVICES={gpu}",
|
||||
"ns-train nerfacto",
|
||||
"--vis wandb",
|
||||
f"--wandb-name {wandb_name}",
|
||||
f"--data data/_prep/images/mipnerf360/processed/{split}",
|
||||
"--pipeline.datamanager.camera-optimizer.mode off",
|
||||
"--trainer.steps-per-eval-all-images 5000",
|
||||
"--trainer.max-num-iterations 250000",
|
||||
"--pipeline.model.use_appearance_embedding False",
|
||||
"nerfstudio-data",
|
||||
"--downscale-factor 4",
|
||||
)
|
||||
)
|
||||
print(command)
|
||||
commands.append(command)
|
||||
# import subprocess
|
||||
# import sys
|
||||
# def f(cmd):
|
||||
# out = subprocess.run(cmd, capture_output=False, shell=True, check=False)
|
||||
# if out.returncode != 0:
|
||||
# print(f"Error running command: {cmd}")
|
||||
# sys.exit(1)
|
||||
# if out.stdout is not None:
|
||||
# return out.stdout.decode("utf-8")
|
||||
|
||||
# from multiprocessing import Pool
|
||||
# with Pool() as p:
|
||||
# print(p.map(f, commands))
|
||||
@@ -235,7 +235,6 @@ class PhototourismDownload(DatasetDownload):
|
||||
shutil.rmtree(tmp_path)
|
||||
os.remove(download_path)
|
||||
|
||||
|
||||
Commands = Union[
|
||||
Annotated[BlenderDownload, tyro.conf.subcommand(name="blender")],
|
||||
Annotated[FriendsDownload, tyro.conf.subcommand(name="friends")],
|
||||
|
||||
Reference in new issue
Block a user