mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-16 04:43:52 +08:00
Added eval script for benchmarking blender dataset (#71)
* fixed train iteration to much longer * done adding test benchmark script
This commit is contained in:
1 parent
e78d7af264
commit
1dc574281c
4 files changed
+104
-31
No files matched your search
@@ -7,6 +7,9 @@ for dataset in ${DATASETS[@]}; do
|
||||
python scripts/run_train.py \
|
||||
data.dataset.data_directory=data/blender/${dataset} \
|
||||
experiment_name=blender_${dataset} \
|
||||
graph.model_dir=mattport_models/ \
|
||||
graph.steps_per_save=25000 \
|
||||
graph.max_num_iterations=2000000 \
|
||||
logging.enable_stats=False \
|
||||
logging.enable_profiler=False &
|
||||
echo "Launched ${dataset} on gpu ${!GPU_IDX}"
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
"""
|
||||
run_benchmark.py
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
|
||||
import torch
|
||||
from hydra import compose, initialize
|
||||
from omegaconf import DictConfig
|
||||
|
||||
from mattport.nerf.trainer import Trainer
|
||||
|
||||
BENCH = {
|
||||
"mic": "outputs/blender_mic/vanilla_nerf/2022-05-11_192109/",
|
||||
"ficus": "outputs/blender_ficus/vanilla_nerf/2022-05-11_192109/",
|
||||
"chair": "outputs/blender_chair/vanilla_nerf/2022-05-11_192109/",
|
||||
"hotdog": "outputs/blender_hotdog/vanilla_nerf/2022-05-11_192109/",
|
||||
"materials": "outputs/blender_materials/vanilla_nerf/2022-05-11_192109/",
|
||||
"drums": "outputs/blender_drums/vanilla_nerf/2022-05-11_192109/",
|
||||
"ship": "outputs/blender_ship/vanilla_nerf/2022-05-11_192109/",
|
||||
"lego": "outputs/blender_lego/vanilla_nerf/2022-05-11_192109/",
|
||||
}
|
||||
|
||||
|
||||
def load_best_ckpt(hydra_dir: str, config: DictConfig):
|
||||
"""helper function to update config with latest checkpoint in specified hydra_dir"""
|
||||
model_dir = os.path.join(hydra_dir, config.graph.model_dir)
|
||||
latest_ckpt = os.listdir(model_dir)
|
||||
latest_ckpt.sort()
|
||||
latest_ckpt = latest_ckpt[-1]
|
||||
step = os.path.splitext(latest_ckpt)[0].split("-")[-1]
|
||||
config.graph.resume_train.load_dir = model_dir
|
||||
config.graph.resume_train.load_step = int(step)
|
||||
config.logging.writer.TensorboardWriter.save_dir = "/tmp/"
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function."""
|
||||
benchmarks = {}
|
||||
for dataset, hydra_dir in BENCH.items():
|
||||
initialize(config_path=os.path.join("../../", hydra_dir, ".hydra/"))
|
||||
config = compose("config.yaml")
|
||||
load_best_ckpt(hydra_dir, config)
|
||||
trainer = Trainer(config, local_rank=0, world_size=1)
|
||||
trainer.setup(test_mode=True)
|
||||
avg_psnr = 0
|
||||
for step, image_idx in enumerate(config.data.validation_image_indices):
|
||||
with torch.no_grad():
|
||||
psnr = trainer.test_image(image_idx=image_idx, step=step)
|
||||
avg_psnr = (step * avg_psnr + psnr) / (step + 1)
|
||||
benchmarks[dataset] = avg_psnr
|
||||
|
||||
benchmark_info = {"bench": BENCH, "results": benchmarks}
|
||||
# TODO(more sophisticated saving to output directory)
|
||||
with open("/tmp/benchmark.json", "w", encoding="utf8") as f:
|
||||
json.dump(benchmark_info, f)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user