updated benchmark scripts (#639)

scripts/benchmarking/launch_eval_blender.sh

@@ -1,32 +1,36 @@
 helpFunction_launch_eval()
 {
-   echo "Usage: $0 -c config_name -g gpu_list"
-   echo -e "\t-c name of method to benchmark (e.g. graph_mipnerf, graph_instant_ngp)"
-   echo -o "\t-o base directory for where all the benchmarks are stored (e.g. outputs/)"
-   echo -e "\t-m month of the benchmark of format xx (e.g. 08)"
-   echo -e "\t-d date of the benchmark of format xx (e.g. 01)"
-   echo -e "\t-y year of the benchmark of format xxxx (e.g. 2022)"
-   echo -e "\t-s second timestamp of the benchmark of format xxxxxx (e.g. 003603)"
-   echo -e "\t-g list of space-separated gpu numbers to launch train on (e.g. 0 2 4 5)"
+   echo "Usage: $0 -m method_name -o output_dir -t timestamp -g [OPTIONAL] gpu_list"
+   echo -e "\t-m name of method to benchmark (e.g. nerfacto, instant-ngp)"
+   echo -e "\t-o base directory for where all the benchmarks are stored (e.g. outputs/)"
+   echo -e "\t-t timstamp, if using launch_train_blender.sh will be of format %Y-%m-%d_%H%M%S"
+   echo -e "\t-g [OPTIONAL] list of space-separated gpu numbers to launch train on (e.g. 0 2 4 5)"
    exit 1 # Exit program after printing help
 }
 
-while getopts "c:o:m:d:y:s:g" opt; do
+while getopts "m:o:t:g" opt; do
     case "$opt" in
-        c ) config_name="$OPTARG" ;;
+        m ) method_name="$OPTARG" ;;
         o ) output_dir="$OPTARG" ;;
-        m ) month="$OPTARG" ;;
-        d ) date="$OPTARG" ;;
-        y ) year="$OPTARG" ;;
-        s ) seconds="$OPTARG" ;;
+        t ) timestamp="$OPTARG" ;;
         g ) gpu_list="$OPTARG" ;;
         ? ) helpFunction_launch_eval ;; 
     esac
 done
 
-if [ -z "$config_name" ]; then
+if [ -z "$method_name" ]; then
     echo "Missing method name"
-    helpFunction_launch_train
+    helpFunction_launch_eval
+fi
+
+if [ -z "$output_dir" ]; then
+    echo "Missing output directory location"
+    helpFunction_launch_eval
+fi
+
+if [ -z "$timestamp" ]; then
+    echo "Missing timestamp specification"
+    helpFunction_launch_eval
 fi
 
 shift $((OPTIND-1))
@@ -57,12 +61,10 @@ len=${#GPU_IDX[@]}
 
 for dataset in ${DATASETS[@]}; do
     export CUDA_VISIBLE_DEVICES=${GPU_IDX[$idx]}
-    base_config_name=${config_name/"graph_"/""}
-    config_path="${output_dir}/blender_${dataset}_${month}-${date}-${year}/${base_config_name}/${year}-${month}-${date}_${seconds}/nerfstudio_models/"
-    python scripts/eval.py compute-psnr \
-        --load-config=${config_path} \
-        --output-path=${output_dir}/${base_config_name}/blender_${dataset}_${month}-${date}-${year}_${seconds}.json &
-    echo "Launched ${checkpoint_dir} on gpu ${GPU_IDX[$idx]}"
+    config_path="${output_dir}/blender_${dataset}_${timestamp::-7}/${method_name}/${timestamp}/config.yml"
+    ns-eval --load-config=${config_path} \
+            --output-path=${output_dir}/${method_name}/blender_${timestamp}.json &
+    echo "Launched ${config_path} on gpu ${GPU_IDX[$idx]}"
 
     # update gpu
     if [ $idx == $len ]; then

scripts/benchmarking/launch_train_blender.sh

@@ -2,21 +2,21 @@
 
 helpFunction_launch_train()
 {
-   echo "Usage: $0 -c config_name -g gpu_list"
-   echo -e "\t-c name of config to benchmark (e.g. mipnerf, instant_ngp)"
-   echo -e "\t-g list of space-separated gpu numbers to launch train on (e.g. 0 2 4 5)"
+   echo "Usage: $0 -m method_name -g [OPTIONAL] gpu_list"
+   echo -e "\t-m name of config to benchmark (e.g. mipnerf, instant_ngp)"
+   echo -e "\t-g [OPTIONAL] list of space-separated gpu numbers to launch train on (e.g. 0 2 4 5)"
    exit 1 # Exit program after printing help
 }
 
-while getopts "c:g" opt; do
+while getopts "m:g" opt; do
     case "$opt" in
-        c ) config_name="$OPTARG" ;;
+        m ) method_name="$OPTARG" ;;
         g ) gpu_list="$OPTARG" ;;
         ? ) helpFunction ;; 
     esac
 done
 
-if [ -z "$config_name" ]; then
+if [ -z "$method_name" ]; then
     echo "Missing method name"
     helpFunction_launch_train
 fi
@@ -45,23 +45,24 @@ echo available gpus... ${GPU_IDX[@]}
 DATASETS=("mic" "ficus" "chair" "hotdog" "materials" "drums" "ship" "lego")
 date
 now=$(date)
-tag=$(date +'%m-%d-%Y')
+tag=$(date +'%Y-%m-%d')
 idx=0
 len=${#GPU_IDX[@]}
 ((len=len-1))
 
 for dataset in ${DATASETS[@]}; do
     export CUDA_VISIBLE_DEVICES=${GPU_IDX[$idx]}
-    python scripts/train.py \
-           ${config_name} \
-           --logging.writer.2.no-enable \
-           --pipeline.datamanager.train-dataset.data-directory=data/blender/${dataset} \
-           --experiment-name=blender_${dataset}_${tag} \
-           --trainer.model-dir=nerfstudio_models/ \
-           --trainer.steps-per-save=16000 \
-           --trainer.max-num-iterations=16500 \
-           --logging.no-enable-profiler &
-    echo "Launched ${config_name} ${dataset} on gpu ${GPU_IDX[$idx]}, ${tag}"
+    ns-train ${method_name} \
+             --data=data/blender/${dataset} \
+             --experiment-name=blender_${dataset}_${tag} \
+             --trainer.relative-model-dir=nerfstudio_models/ \
+             --trainer.steps-per-save=1000 \
+             --trainer.max-num-iterations=16500 \
+             --logging.local-writer.no-enable  \
+             --logging.no-enable-profiler \
+             --vis wandb \
+             blender-data &
+    echo "Launched ${method_name} ${dataset} on gpu ${GPU_IDX[$idx]}, ${tag}"
     
     # update gpu
     if [ $idx == $len ]; then