mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-20 06:35:36 +08:00
benchmark scripts
This commit is contained in:
1 parent
4cad94a5ba
commit
5cacc91142
2 files changed
+173
-17
No files matched your search
@@ -21,12 +21,12 @@ mipnerf360_table_rows = [
|
||||
"nerfacto",
|
||||
"--pipeline.eval_optimize_cameras True --pipeline.eval_optimize_appearance False --pipeline.datamanager.camera-optimizer.mode off --pipeline.model.use-appearance-embedding False nerfstudio-data --downscale-factor 4 --train-split-percentage 0.875",
|
||||
),
|
||||
# TODO: instant-ngp method
|
||||
# (
|
||||
# "instant-ngp-w/o-pose-app",
|
||||
# "instant-ngp",
|
||||
# "--pipeline.eval_optimize_cameras True --pipeline.eval_optimize_appearance False --pipeline.datamanager.camera-optimizer.mode off --pipeline.model.use-appearance-embedding False nerfstudio-data --downscale-factor 4 --train-split-percentage 0.875",
|
||||
# ),
|
||||
# instant-ngp method
|
||||
(
|
||||
"instant-ngp-w/o-pose-app",
|
||||
"instant-ngp",
|
||||
"--pipeline.eval_optimize_cameras True --pipeline.eval_optimize_appearance False --pipeline.datamanager.camera-optimizer.mode off --pipeline.model.use-appearance-embedding False nerfstudio-data --downscale-factor 4 --train-split-percentage 0.875",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@@ -133,7 +133,7 @@ def main(capture_names, table_rows, data_path: Path = Path("data/nerfstudio")):
|
||||
|
||||
def task():
|
||||
print("Starting command: ", command)
|
||||
out = run_command(command, verbose=False)
|
||||
# out = run_command(command, verbose=False)
|
||||
# time.sleep(5)
|
||||
print("Finished command: ", command)
|
||||
|
||||
@@ -152,5 +152,6 @@ def main(capture_names, table_rows, data_path: Path = Path("data/nerfstudio")):
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(mipnerf360_capture_names, mipnerf360_table_rows, data_path=Path("data/nerfstudio-data-mipnerf360"))
|
||||
pass
|
||||
# main(mipnerf360_capture_names, mipnerf360_table_rows, data_path=Path("data/nerfstudio-data-mipnerf360"))
|
||||
# main(ablations_capture_names, ablations_table_rows)
|
||||
@@ -2,7 +2,7 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -12,13 +12,26 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "ImportError",
|
||||
"evalue": "cannot import name 'table_rows' from 'benchmark_nerfstudio_paper' (/home/ethanweber/nerfactory/scripts/benchmarking/benchmark_nerfstudio_paper.py)",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[0;32mIn [4], line 4\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mcopy\u001b[39;00m\n\u001b[1;32m 3\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mpprint\u001b[39;00m\n\u001b[0;32m----> 4\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mbenchmark_nerfstudio_paper\u001b[39;00m \u001b[39mimport\u001b[39;00m table_rows\n\u001b[1;32m 6\u001b[0m \u001b[39m# methods sorted in the same order we want in the paper tables\u001b[39;00m\n\u001b[1;32m 7\u001b[0m methods \u001b[39m=\u001b[39m [row[\u001b[39m0\u001b[39m] \u001b[39mfor\u001b[39;00m row \u001b[39min\u001b[39;00m table_rows]\n",
|
||||
"\u001b[0;31mImportError\u001b[0m: cannot import name 'table_rows' from 'benchmark_nerfstudio_paper' (/home/ethanweber/nerfactory/scripts/benchmarking/benchmark_nerfstudio_paper.py)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import copy\n",
|
||||
"from benchmark_nerfacto import table_rows\n",
|
||||
"import pprint\n",
|
||||
"from benchmark_nerfstudio_paper import table_rows\n",
|
||||
"\n",
|
||||
"# methods sorted in the same order we want in the paper tables\n",
|
||||
"methods = [row[0] for row in table_rows]"
|
||||
@@ -26,21 +39,31 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def csv_filename_to_results(csv_filename, num_decimals=2):\n",
|
||||
"def csv_filename_to_results(csv_filename, num_decimals=2, num_rows_to_consider=None):\n",
|
||||
" # code to read the csv file\n",
|
||||
" # and return a dictionary\n",
|
||||
"\n",
|
||||
" df = pd.read_csv(csv_filename)\n",
|
||||
"\n",
|
||||
" column_titles = list(df.columns.values)\n",
|
||||
" last_row = list(df.values[-1])\n",
|
||||
"\n",
|
||||
" # print(len(column_titles))\n",
|
||||
" # print(len(last_row))\n",
|
||||
" # iterate as long as the value is not None\n",
|
||||
" if num_rows_to_consider is None:\n",
|
||||
" rows = list(df.values)\n",
|
||||
" else:\n",
|
||||
" rows = list(df.values)[:num_rows_to_consider]\n",
|
||||
" valid_row = list(df.values[-1])\n",
|
||||
" for row in rows:\n",
|
||||
" for i in range(len(row)):\n",
|
||||
" value = row[i]\n",
|
||||
" if not pd.isnull(value):\n",
|
||||
" valid_row[i] = value\n",
|
||||
" last_row = valid_row\n",
|
||||
" # last_row will be the row that's valid\n",
|
||||
"\n",
|
||||
" from collections import defaultdict\n",
|
||||
" results = defaultdict(dict)\n",
|
||||
@@ -157,6 +180,138 @@
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"csv_filename_psrn = \"/home/ethanweber/nerfactory/mipnerf360-instant-ngp-psnr.csv\"\n",
|
||||
"csv_filename_ssim = \"/home/ethanweber/nerfactory/mipnerf360-instant-ngp-ssim.csv\"\n",
|
||||
"csv_filename_lpips = \"/home/ethanweber/nerfactory/mipnerf360-instant-ngp-lpips.csv\"\n",
|
||||
"\n",
|
||||
"results_psrn = csv_filename_to_results(csv_filename_psrn, num_rows_to_consider=1)\n",
|
||||
"results_ssim = csv_filename_to_results(csv_filename_ssim, num_decimals=3, num_rows_to_consider=1)\n",
|
||||
"results_lpips = csv_filename_to_results(csv_filename_lpips, num_decimals=3, num_rows_to_consider=1)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"psnr\n",
|
||||
"[('bicycle', {'instant-ngp-w/o-pose-app': '19.92'}),\n",
|
||||
" ('garden', {'instant-ngp-w/o-pose-app': '22.68'}),\n",
|
||||
" ('stump', {'instant-ngp-w/o-pose-app': '20.25'}),\n",
|
||||
" ('room', {'instant-ngp-w/o-pose-app': '28.04'}),\n",
|
||||
" ('counter', {'instant-ngp-w/o-pose-app': '23.59'}),\n",
|
||||
" ('kitchen', {'instant-ngp-w/o-pose-app': '24.61'}),\n",
|
||||
" ('bonsai', {'instant-ngp-w/o-pose-app': '25.67'})]\n",
|
||||
"\n",
|
||||
"ssim\n",
|
||||
"[('bicycle', {'instant-ngp-w/o-pose-app': '0.448'}),\n",
|
||||
" ('garden', {'instant-ngp-w/o-pose-app': '0.542'}),\n",
|
||||
" ('stump', {'instant-ngp-w/o-pose-app': '0.465'}),\n",
|
||||
" ('room', {'instant-ngp-w/o-pose-app': '0.828'}),\n",
|
||||
" ('counter', {'instant-ngp-w/o-pose-app': '0.701'}),\n",
|
||||
" ('kitchen', {'instant-ngp-w/o-pose-app': '0.642'}),\n",
|
||||
" ('bonsai', {'instant-ngp-w/o-pose-app': '0.792'})]\n",
|
||||
"\n",
|
||||
"lpips\n",
|
||||
"[('bicycle', {'instant-ngp-w/o-pose-app': '0.449'}),\n",
|
||||
" ('garden', {'instant-ngp-w/o-pose-app': '0.402'}),\n",
|
||||
" ('stump', {'instant-ngp-w/o-pose-app': '0.440'}),\n",
|
||||
" ('room', {'instant-ngp-w/o-pose-app': '0.170'}),\n",
|
||||
" ('counter', {'instant-ngp-w/o-pose-app': '0.271'}),\n",
|
||||
" ('kitchen', {'instant-ngp-w/o-pose-app': '0.281'}),\n",
|
||||
" ('bonsai', {'instant-ngp-w/o-pose-app': '0.186'})]\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"mipnerf360_nerfacto_capture_names = [\"bicycle\", \"garden\", \"stump\", \"room\", \"counter\", \"kitchen\", \"bonsai\"]\n",
|
||||
"\n",
|
||||
"print(\"psnr\")\n",
|
||||
"s = [(key, results_psrn[key]) for key in mipnerf360_nerfacto_capture_names]\n",
|
||||
"pprint.pprint(s)\n",
|
||||
"print()\n",
|
||||
"\n",
|
||||
"print(\"ssim\")\n",
|
||||
"s = [(key, results_ssim[key]) for key in mipnerf360_nerfacto_capture_names]\n",
|
||||
"pprint.pprint(s)\n",
|
||||
"print()\n",
|
||||
"\n",
|
||||
"print(\"lpips\")\n",
|
||||
"s = [(key, results_lpips[key]) for key in mipnerf360_nerfacto_capture_names]\n",
|
||||
"pprint.pprint(s)\n",
|
||||
"print()\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"psnr\n",
|
||||
"[('bicycle', {'instant-ngp-w/o-pose-app': '18.78'}),\n",
|
||||
" ('garden', {'instant-ngp-w/o-pose-app': '20.82'}),\n",
|
||||
" ('stump', {'instant-ngp-w/o-pose-app': '19.73'}),\n",
|
||||
" ('room', {'instant-ngp-w/o-pose-app': '25.26'}),\n",
|
||||
" ('counter', {'instant-ngp-w/o-pose-app': '22.09'}),\n",
|
||||
" ('kitchen', {'instant-ngp-w/o-pose-app': '22.82'}),\n",
|
||||
" ('bonsai', {'instant-ngp-w/o-pose-app': '23.86'})]\n",
|
||||
"\n",
|
||||
"ssim\n",
|
||||
"[('bicycle', {'instant-ngp-w/o-pose-app': '0.389'}),\n",
|
||||
" ('garden', {'instant-ngp-w/o-pose-app': '0.390'}),\n",
|
||||
" ('stump', {'instant-ngp-w/o-pose-app': '0.408'}),\n",
|
||||
" ('room', {'instant-ngp-w/o-pose-app': '0.749'}),\n",
|
||||
" ('counter', {'instant-ngp-w/o-pose-app': '0.621'}),\n",
|
||||
" ('kitchen', {'instant-ngp-w/o-pose-app': '0.530'}),\n",
|
||||
" ('bonsai', {'instant-ngp-w/o-pose-app': '0.665'})]\n",
|
||||
"\n",
|
||||
"lpips\n",
|
||||
"[('bicycle', {'instant-ngp-w/o-pose-app': '0.620'}),\n",
|
||||
" ('garden', {'instant-ngp-w/o-pose-app': '0.619'}),\n",
|
||||
" ('stump', {'instant-ngp-w/o-pose-app': '0.607'}),\n",
|
||||
" ('room', {'instant-ngp-w/o-pose-app': '0.311'}),\n",
|
||||
" ('counter', {'instant-ngp-w/o-pose-app': '0.417'}),\n",
|
||||
" ('kitchen', {'instant-ngp-w/o-pose-app': '0.427'}),\n",
|
||||
" ('bonsai', {'instant-ngp-w/o-pose-app': '0.382'})]\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"mipnerf360_nerfacto_capture_names = [\"bicycle\", \"garden\", \"stump\", \"room\", \"counter\", \"kitchen\", \"bonsai\"]\n",
|
||||
"\n",
|
||||
"print(\"psnr\")\n",
|
||||
"s = [(key, results_psrn[key]) for key in mipnerf360_nerfacto_capture_names]\n",
|
||||
"pprint.pprint(s)\n",
|
||||
"print()\n",
|
||||
"\n",
|
||||
"print(\"ssim\")\n",
|
||||
"s = [(key, results_ssim[key]) for key in mipnerf360_nerfacto_capture_names]\n",
|
||||
"pprint.pprint(s)\n",
|
||||
"print()\n",
|
||||
"\n",
|
||||
"print(\"lpips\")\n",
|
||||
"s = [(key, results_lpips[key]) for key in mipnerf360_nerfacto_capture_names]\n",
|
||||
"pprint.pprint(s)\n",
|
||||
"print()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
|
||||
Reference in new issue
Block a user