From 5cacc911426a132823d457ac1ee4832f3f37f344 Mon Sep 17 00:00:00 2001 From: Ethan Weber Date: Tue, 10 Jan 2023 23:22:20 +0000 Subject: [PATCH] benchmark scripts --- .../benchmark_nerfstudio_paper.py | 17 +- scripts/benchmarking/create_tables.ipynb | 173 +++++++++++++++++- 2 files changed, 173 insertions(+), 17 deletions(-) diff --git a/scripts/benchmarking/benchmark_nerfstudio_paper.py b/scripts/benchmarking/benchmark_nerfstudio_paper.py index 742e825f9..34081620d 100644 --- a/scripts/benchmarking/benchmark_nerfstudio_paper.py +++ b/scripts/benchmarking/benchmark_nerfstudio_paper.py @@ -21,12 +21,12 @@ mipnerf360_table_rows = [ "nerfacto", "--pipeline.eval_optimize_cameras True --pipeline.eval_optimize_appearance False --pipeline.datamanager.camera-optimizer.mode off --pipeline.model.use-appearance-embedding False nerfstudio-data --downscale-factor 4 --train-split-percentage 0.875", ), - # TODO: instant-ngp method - # ( - # "instant-ngp-w/o-pose-app", - # "instant-ngp", - # "--pipeline.eval_optimize_cameras True --pipeline.eval_optimize_appearance False --pipeline.datamanager.camera-optimizer.mode off --pipeline.model.use-appearance-embedding False nerfstudio-data --downscale-factor 4 --train-split-percentage 0.875", - # ), + # instant-ngp method + ( + "instant-ngp-w/o-pose-app", + "instant-ngp", + "--pipeline.eval_optimize_cameras True --pipeline.eval_optimize_appearance False --pipeline.datamanager.camera-optimizer.mode off --pipeline.model.use-appearance-embedding False nerfstudio-data --downscale-factor 4 --train-split-percentage 0.875", + ), ] @@ -133,7 +133,7 @@ def main(capture_names, table_rows, data_path: Path = Path("data/nerfstudio")): def task(): print("Starting command: ", command) - out = run_command(command, verbose=False) + # out = run_command(command, verbose=False) # time.sleep(5) print("Finished command: ", command) @@ -152,5 +152,6 @@ def main(capture_names, table_rows, data_path: Path = Path("data/nerfstudio")): if __name__ == "__main__": - main(mipnerf360_capture_names, mipnerf360_table_rows, data_path=Path("data/nerfstudio-data-mipnerf360")) + pass + # main(mipnerf360_capture_names, mipnerf360_table_rows, data_path=Path("data/nerfstudio-data-mipnerf360")) # main(ablations_capture_names, ablations_table_rows) diff --git a/scripts/benchmarking/create_tables.ipynb b/scripts/benchmarking/create_tables.ipynb index 2ad5ad562..8fd94ee0b 100644 --- a/scripts/benchmarking/create_tables.ipynb +++ b/scripts/benchmarking/create_tables.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -12,13 +12,26 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "ImportError", + "evalue": "cannot import name 'table_rows' from 'benchmark_nerfstudio_paper' (/home/ethanweber/nerfactory/scripts/benchmarking/benchmark_nerfstudio_paper.py)", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn [4], line 4\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mcopy\u001b[39;00m\n\u001b[1;32m 3\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mpprint\u001b[39;00m\n\u001b[0;32m----> 4\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mbenchmark_nerfstudio_paper\u001b[39;00m \u001b[39mimport\u001b[39;00m table_rows\n\u001b[1;32m 6\u001b[0m \u001b[39m# methods sorted in the same order we want in the paper tables\u001b[39;00m\n\u001b[1;32m 7\u001b[0m methods \u001b[39m=\u001b[39m [row[\u001b[39m0\u001b[39m] \u001b[39mfor\u001b[39;00m row \u001b[39min\u001b[39;00m table_rows]\n", + "\u001b[0;31mImportError\u001b[0m: cannot import name 'table_rows' from 'benchmark_nerfstudio_paper' (/home/ethanweber/nerfactory/scripts/benchmarking/benchmark_nerfstudio_paper.py)" + ] + } + ], "source": [ "import pandas as pd\n", "import copy\n", - "from benchmark_nerfacto import table_rows\n", + "import pprint\n", + "from benchmark_nerfstudio_paper import table_rows\n", "\n", "# methods sorted in the same order we want in the paper tables\n", "methods = [row[0] for row in table_rows]" @@ -26,21 +39,31 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ - "def csv_filename_to_results(csv_filename, num_decimals=2):\n", + "def csv_filename_to_results(csv_filename, num_decimals=2, num_rows_to_consider=None):\n", " # code to read the csv file\n", " # and return a dictionary\n", "\n", " df = pd.read_csv(csv_filename)\n", "\n", " column_titles = list(df.columns.values)\n", - " last_row = list(df.values[-1])\n", "\n", - " # print(len(column_titles))\n", - " # print(len(last_row))\n", + " # iterate as long as the value is not None\n", + " if num_rows_to_consider is None:\n", + " rows = list(df.values)\n", + " else:\n", + " rows = list(df.values)[:num_rows_to_consider]\n", + " valid_row = list(df.values[-1])\n", + " for row in rows:\n", + " for i in range(len(row)):\n", + " value = row[i]\n", + " if not pd.isnull(value):\n", + " valid_row[i] = value\n", + " last_row = valid_row\n", + " # last_row will be the row that's valid\n", "\n", " from collections import defaultdict\n", " results = defaultdict(dict)\n", @@ -157,6 +180,138 @@ "\n" ] }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "csv_filename_psrn = \"/home/ethanweber/nerfactory/mipnerf360-instant-ngp-psnr.csv\"\n", + "csv_filename_ssim = \"/home/ethanweber/nerfactory/mipnerf360-instant-ngp-ssim.csv\"\n", + "csv_filename_lpips = \"/home/ethanweber/nerfactory/mipnerf360-instant-ngp-lpips.csv\"\n", + "\n", + "results_psrn = csv_filename_to_results(csv_filename_psrn, num_rows_to_consider=1)\n", + "results_ssim = csv_filename_to_results(csv_filename_ssim, num_decimals=3, num_rows_to_consider=1)\n", + "results_lpips = csv_filename_to_results(csv_filename_lpips, num_decimals=3, num_rows_to_consider=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "psnr\n", + "[('bicycle', {'instant-ngp-w/o-pose-app': '19.92'}),\n", + " ('garden', {'instant-ngp-w/o-pose-app': '22.68'}),\n", + " ('stump', {'instant-ngp-w/o-pose-app': '20.25'}),\n", + " ('room', {'instant-ngp-w/o-pose-app': '28.04'}),\n", + " ('counter', {'instant-ngp-w/o-pose-app': '23.59'}),\n", + " ('kitchen', {'instant-ngp-w/o-pose-app': '24.61'}),\n", + " ('bonsai', {'instant-ngp-w/o-pose-app': '25.67'})]\n", + "\n", + "ssim\n", + "[('bicycle', {'instant-ngp-w/o-pose-app': '0.448'}),\n", + " ('garden', {'instant-ngp-w/o-pose-app': '0.542'}),\n", + " ('stump', {'instant-ngp-w/o-pose-app': '0.465'}),\n", + " ('room', {'instant-ngp-w/o-pose-app': '0.828'}),\n", + " ('counter', {'instant-ngp-w/o-pose-app': '0.701'}),\n", + " ('kitchen', {'instant-ngp-w/o-pose-app': '0.642'}),\n", + " ('bonsai', {'instant-ngp-w/o-pose-app': '0.792'})]\n", + "\n", + "lpips\n", + "[('bicycle', {'instant-ngp-w/o-pose-app': '0.449'}),\n", + " ('garden', {'instant-ngp-w/o-pose-app': '0.402'}),\n", + " ('stump', {'instant-ngp-w/o-pose-app': '0.440'}),\n", + " ('room', {'instant-ngp-w/o-pose-app': '0.170'}),\n", + " ('counter', {'instant-ngp-w/o-pose-app': '0.271'}),\n", + " ('kitchen', {'instant-ngp-w/o-pose-app': '0.281'}),\n", + " ('bonsai', {'instant-ngp-w/o-pose-app': '0.186'})]\n", + "\n" + ] + } + ], + "source": [ + "mipnerf360_nerfacto_capture_names = [\"bicycle\", \"garden\", \"stump\", \"room\", \"counter\", \"kitchen\", \"bonsai\"]\n", + "\n", + "print(\"psnr\")\n", + "s = [(key, results_psrn[key]) for key in mipnerf360_nerfacto_capture_names]\n", + "pprint.pprint(s)\n", + "print()\n", + "\n", + "print(\"ssim\")\n", + "s = [(key, results_ssim[key]) for key in mipnerf360_nerfacto_capture_names]\n", + "pprint.pprint(s)\n", + "print()\n", + "\n", + "print(\"lpips\")\n", + "s = [(key, results_lpips[key]) for key in mipnerf360_nerfacto_capture_names]\n", + "pprint.pprint(s)\n", + "print()\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "psnr\n", + "[('bicycle', {'instant-ngp-w/o-pose-app': '18.78'}),\n", + " ('garden', {'instant-ngp-w/o-pose-app': '20.82'}),\n", + " ('stump', {'instant-ngp-w/o-pose-app': '19.73'}),\n", + " ('room', {'instant-ngp-w/o-pose-app': '25.26'}),\n", + " ('counter', {'instant-ngp-w/o-pose-app': '22.09'}),\n", + " ('kitchen', {'instant-ngp-w/o-pose-app': '22.82'}),\n", + " ('bonsai', {'instant-ngp-w/o-pose-app': '23.86'})]\n", + "\n", + "ssim\n", + "[('bicycle', {'instant-ngp-w/o-pose-app': '0.389'}),\n", + " ('garden', {'instant-ngp-w/o-pose-app': '0.390'}),\n", + " ('stump', {'instant-ngp-w/o-pose-app': '0.408'}),\n", + " ('room', {'instant-ngp-w/o-pose-app': '0.749'}),\n", + " ('counter', {'instant-ngp-w/o-pose-app': '0.621'}),\n", + " ('kitchen', {'instant-ngp-w/o-pose-app': '0.530'}),\n", + " ('bonsai', {'instant-ngp-w/o-pose-app': '0.665'})]\n", + "\n", + "lpips\n", + "[('bicycle', {'instant-ngp-w/o-pose-app': '0.620'}),\n", + " ('garden', {'instant-ngp-w/o-pose-app': '0.619'}),\n", + " ('stump', {'instant-ngp-w/o-pose-app': '0.607'}),\n", + " ('room', {'instant-ngp-w/o-pose-app': '0.311'}),\n", + " ('counter', {'instant-ngp-w/o-pose-app': '0.417'}),\n", + " ('kitchen', {'instant-ngp-w/o-pose-app': '0.427'}),\n", + " ('bonsai', {'instant-ngp-w/o-pose-app': '0.382'})]\n", + "\n" + ] + } + ], + "source": [ + "mipnerf360_nerfacto_capture_names = [\"bicycle\", \"garden\", \"stump\", \"room\", \"counter\", \"kitchen\", \"bonsai\"]\n", + "\n", + "print(\"psnr\")\n", + "s = [(key, results_psrn[key]) for key in mipnerf360_nerfacto_capture_names]\n", + "pprint.pprint(s)\n", + "print()\n", + "\n", + "print(\"ssim\")\n", + "s = [(key, results_ssim[key]) for key in mipnerf360_nerfacto_capture_names]\n", + "pprint.pprint(s)\n", + "print()\n", + "\n", + "print(\"lpips\")\n", + "s = [(key, results_lpips[key]) for key in mipnerf360_nerfacto_capture_names]\n", + "pprint.pprint(s)\n", + "print()" + ] + }, { "cell_type": "code", "execution_count": null,