Printing profile on sig kill. (#44)

* added skeleton for terminal logger

* done skeleton terminal writer logic... todo: actual calls to the write classes

* moved lambda to top function

* added logic to support timing

* prettier printing

* prettier printing

* lint

* separated out timing and other stats variables. printing in dict format

* keeping logging history and wiping rest

* updated logic to calculate total times instead of average times

* made stats tracker configurable

* moved stats tracker and profiler to own class file; implemented basic functionality for profiler

* lint removing duplicate function

* update readme; moved around config variables

* profiler prints on sig kill

scripts/run_train.py

@@ -16,6 +16,7 @@ import torch.multiprocessing as mp
 from omegaconf import DictConfig
 
 from mattport.nerf.trainer import Trainer
+from mattport.utils import profiler
 
 logging.basicConfig(format="[%(filename)s:%(lineno)d] %(message)s", level=logging.DEBUG)
 
@@ -174,8 +175,17 @@ def launch(
                     process.terminate()
                 process.join()
                 logger.info("Process %s finished", str(i))
+        finally:
+            if config.logging_configs.enable_profiler:
+                profiler.PROFILER.print_profile()
     else:
-        main_func(local_rank=0, world_size=1, config=config)
+        try:
+            main_func(local_rank=0, world_size=1, config=config)
+        except KeyboardInterrupt:
+            pass
+        finally:
+            if config.logging_configs.enable_profiler:
+                profiler.PROFILER.print_profile()
 
 
 @hydra.main(config_path="../configs", config_name="default.yaml")