From 3df29717dd4e957db0bfc1386fe22638a677077a Mon Sep 17 00:00:00 2001
From: evonneng <evonneng@users.noreply.github.com>
Date: Mon, 18 Jul 2022 12:56:32 -0700
Subject: [PATCH] added avg total fps (#170)

* added fps as benchmarking stats

* update total avg for fps/rays/s
---
 scripts/benchmarking/run_benchmark.py | 15 +++++++++++++++
 scripts/run_eval.py                   |  1 +
 2 files changed, 16 insertions(+)

diff --git a/scripts/benchmarking/run_benchmark.py b/scripts/benchmarking/run_benchmark.py
index 739d37767..797b95b69 100644
--- a/scripts/benchmarking/run_benchmark.py
+++ b/scripts/benchmarking/run_benchmark.py
@@ -5,9 +5,11 @@ import argparse
 import json
 import logging
 import os
+from typing import Any, Dict
 
 from hydra import compose, initialize
 from hydra.core.global_hydra import GlobalHydra
+import numpy as np
 from omegaconf import DictConfig
 from tqdm import tqdm
 
@@ -58,6 +60,14 @@ def _load_hydra_config(hydra_dir: str) -> DictConfig:
     return config
 
 
+def _calc_avg(stat_name: str, benchmark: Dict[str, Any]):
+    """helper to calculate the average across all objects in dataset"""
+    stats = []
+    for _, object_stats in benchmark.items():
+        stats.append(object_stats[stat_name])
+    return np.mean(stats)
+
+
 def main(args):
     """Main function."""
     benchmarks = {}
@@ -75,9 +85,14 @@ def main(args):
         # reset hydra config
         GlobalHydra.instance().clear()
 
+    avg_rays_per_sec = _calc_avg("avg rays per sec", benchmarks)
+    avg_fps = _calc_avg("avg fps", benchmarks)
+
     # output benchmark statistics to a json file
     benchmark_info = {
         "meta info": {"graph": args.graph, "benchmark_date": args.benchmark_date, "hydra": args.hydra_base_dir},
+        "avg rays per sec": avg_rays_per_sec,
+        "avg fps": avg_fps,
         "results": benchmarks,
     }
     json_file = os.path.join(args.hydra_base_dir, f"{args.benchmark_date}.json")
diff --git a/scripts/run_eval.py b/scripts/run_eval.py
index 30df3ca14..f51b21742 100644
--- a/scripts/run_eval.py
+++ b/scripts/run_eval.py
@@ -66,6 +66,7 @@ def run_inference(config: DictConfig, local_rank: int = 0, world_size: int = 1)
     dataset_inputs_train, _ = setup_dataset_train(config.data, device=device)
     _, dataloader_eval = setup_dataset_eval(config.data, test_mode=True, device=device)
     graph = setup_graph(config.graph, dataset_inputs_train, device=device)
+    graph.eval()
 
     # load checkpointed information
     _load_checkpoint(config.trainer.resume_train, graph)