From 2de5afae0c5547a5749d64d0f2b539faa147bce9 Mon Sep 17 00:00:00 2001 From: evonneng Date: Mon, 3 Oct 2022 21:42:31 -0700 Subject: [PATCH] updated benchmark scripts (#639) --- scripts/benchmarking/launch_eval_blender.sh | 46 ++++++++++---------- scripts/benchmarking/launch_train_blender.sh | 35 +++++++-------- 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/scripts/benchmarking/launch_eval_blender.sh b/scripts/benchmarking/launch_eval_blender.sh index 7a0c9c878..2c79856a5 100755 --- a/scripts/benchmarking/launch_eval_blender.sh +++ b/scripts/benchmarking/launch_eval_blender.sh @@ -1,32 +1,36 @@ helpFunction_launch_eval() { - echo "Usage: $0 -c config_name -g gpu_list" - echo -e "\t-c name of method to benchmark (e.g. graph_mipnerf, graph_instant_ngp)" - echo -o "\t-o base directory for where all the benchmarks are stored (e.g. outputs/)" - echo -e "\t-m month of the benchmark of format xx (e.g. 08)" - echo -e "\t-d date of the benchmark of format xx (e.g. 01)" - echo -e "\t-y year of the benchmark of format xxxx (e.g. 2022)" - echo -e "\t-s second timestamp of the benchmark of format xxxxxx (e.g. 003603)" - echo -e "\t-g list of space-separated gpu numbers to launch train on (e.g. 0 2 4 5)" + echo "Usage: $0 -m method_name -o output_dir -t timestamp -g [OPTIONAL] gpu_list" + echo -e "\t-m name of method to benchmark (e.g. nerfacto, instant-ngp)" + echo -e "\t-o base directory for where all the benchmarks are stored (e.g. outputs/)" + echo -e "\t-t timstamp, if using launch_train_blender.sh will be of format %Y-%m-%d_%H%M%S" + echo -e "\t-g [OPTIONAL] list of space-separated gpu numbers to launch train on (e.g. 0 2 4 5)" exit 1 # Exit program after printing help } -while getopts "c:o:m:d:y:s:g" opt; do +while getopts "m:o:t:g" opt; do case "$opt" in - c ) config_name="$OPTARG" ;; + m ) method_name="$OPTARG" ;; o ) output_dir="$OPTARG" ;; - m ) month="$OPTARG" ;; - d ) date="$OPTARG" ;; - y ) year="$OPTARG" ;; - s ) seconds="$OPTARG" ;; + t ) timestamp="$OPTARG" ;; g ) gpu_list="$OPTARG" ;; ? ) helpFunction_launch_eval ;; esac done -if [ -z "$config_name" ]; then +if [ -z "$method_name" ]; then echo "Missing method name" - helpFunction_launch_train + helpFunction_launch_eval +fi + +if [ -z "$output_dir" ]; then + echo "Missing output directory location" + helpFunction_launch_eval +fi + +if [ -z "$timestamp" ]; then + echo "Missing timestamp specification" + helpFunction_launch_eval fi shift $((OPTIND-1)) @@ -57,12 +61,10 @@ len=${#GPU_IDX[@]} for dataset in ${DATASETS[@]}; do export CUDA_VISIBLE_DEVICES=${GPU_IDX[$idx]} - base_config_name=${config_name/"graph_"/""} - config_path="${output_dir}/blender_${dataset}_${month}-${date}-${year}/${base_config_name}/${year}-${month}-${date}_${seconds}/nerfstudio_models/" - python scripts/eval.py compute-psnr \ - --load-config=${config_path} \ - --output-path=${output_dir}/${base_config_name}/blender_${dataset}_${month}-${date}-${year}_${seconds}.json & - echo "Launched ${checkpoint_dir} on gpu ${GPU_IDX[$idx]}" + config_path="${output_dir}/blender_${dataset}_${timestamp::-7}/${method_name}/${timestamp}/config.yml" + ns-eval --load-config=${config_path} \ + --output-path=${output_dir}/${method_name}/blender_${timestamp}.json & + echo "Launched ${config_path} on gpu ${GPU_IDX[$idx]}" # update gpu if [ $idx == $len ]; then diff --git a/scripts/benchmarking/launch_train_blender.sh b/scripts/benchmarking/launch_train_blender.sh index 9e1714e41..532c3c665 100755 --- a/scripts/benchmarking/launch_train_blender.sh +++ b/scripts/benchmarking/launch_train_blender.sh @@ -2,21 +2,21 @@ helpFunction_launch_train() { - echo "Usage: $0 -c config_name -g gpu_list" - echo -e "\t-c name of config to benchmark (e.g. mipnerf, instant_ngp)" - echo -e "\t-g list of space-separated gpu numbers to launch train on (e.g. 0 2 4 5)" + echo "Usage: $0 -m method_name -g [OPTIONAL] gpu_list" + echo -e "\t-m name of config to benchmark (e.g. mipnerf, instant_ngp)" + echo -e "\t-g [OPTIONAL] list of space-separated gpu numbers to launch train on (e.g. 0 2 4 5)" exit 1 # Exit program after printing help } -while getopts "c:g" opt; do +while getopts "m:g" opt; do case "$opt" in - c ) config_name="$OPTARG" ;; + m ) method_name="$OPTARG" ;; g ) gpu_list="$OPTARG" ;; ? ) helpFunction ;; esac done -if [ -z "$config_name" ]; then +if [ -z "$method_name" ]; then echo "Missing method name" helpFunction_launch_train fi @@ -45,23 +45,24 @@ echo available gpus... ${GPU_IDX[@]} DATASETS=("mic" "ficus" "chair" "hotdog" "materials" "drums" "ship" "lego") date now=$(date) -tag=$(date +'%m-%d-%Y') +tag=$(date +'%Y-%m-%d') idx=0 len=${#GPU_IDX[@]} ((len=len-1)) for dataset in ${DATASETS[@]}; do export CUDA_VISIBLE_DEVICES=${GPU_IDX[$idx]} - python scripts/train.py \ - ${config_name} \ - --logging.writer.2.no-enable \ - --pipeline.datamanager.train-dataset.data-directory=data/blender/${dataset} \ - --experiment-name=blender_${dataset}_${tag} \ - --trainer.model-dir=nerfstudio_models/ \ - --trainer.steps-per-save=16000 \ - --trainer.max-num-iterations=16500 \ - --logging.no-enable-profiler & - echo "Launched ${config_name} ${dataset} on gpu ${GPU_IDX[$idx]}, ${tag}" + ns-train ${method_name} \ + --data=data/blender/${dataset} \ + --experiment-name=blender_${dataset}_${tag} \ + --trainer.relative-model-dir=nerfstudio_models/ \ + --trainer.steps-per-save=1000 \ + --trainer.max-num-iterations=16500 \ + --logging.local-writer.no-enable \ + --logging.no-enable-profiler \ + --vis wandb \ + blender-data & + echo "Launched ${method_name} ${dataset} on gpu ${GPU_IDX[$idx]}, ${tag}" # update gpu if [ $idx == $len ]; then