mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-11 10:34:29 +08:00
351 lines
14 KiB
Plaintext
351 lines
14 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"%load_ext autoreload\n",
|
|
"%autoreload 2"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"ename": "ImportError",
|
|
"evalue": "cannot import name 'table_rows' from 'benchmark_nerfstudio_paper' (/home/ethanweber/nerfactory/scripts/benchmarking/benchmark_nerfstudio_paper.py)",
|
|
"output_type": "error",
|
|
"traceback": [
|
|
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
|
"\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)",
|
|
"Cell \u001b[0;32mIn [4], line 4\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mcopy\u001b[39;00m\n\u001b[1;32m 3\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mpprint\u001b[39;00m\n\u001b[0;32m----> 4\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mbenchmark_nerfstudio_paper\u001b[39;00m \u001b[39mimport\u001b[39;00m table_rows\n\u001b[1;32m 6\u001b[0m \u001b[39m# methods sorted in the same order we want in the paper tables\u001b[39;00m\n\u001b[1;32m 7\u001b[0m methods \u001b[39m=\u001b[39m [row[\u001b[39m0\u001b[39m] \u001b[39mfor\u001b[39;00m row \u001b[39min\u001b[39;00m table_rows]\n",
|
|
"\u001b[0;31mImportError\u001b[0m: cannot import name 'table_rows' from 'benchmark_nerfstudio_paper' (/home/ethanweber/nerfactory/scripts/benchmarking/benchmark_nerfstudio_paper.py)"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"import copy\n",
|
|
"import pprint\n",
|
|
"from benchmark_nerfstudio_paper import table_rows\n",
|
|
"\n",
|
|
"# methods sorted in the same order we want in the paper tables\n",
|
|
"methods = [row[0] for row in table_rows]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def csv_filename_to_results(csv_filename, num_decimals=2, num_rows_to_consider=None):\n",
|
|
" # code to read the csv file\n",
|
|
" # and return a dictionary\n",
|
|
"\n",
|
|
" df = pd.read_csv(csv_filename)\n",
|
|
"\n",
|
|
" column_titles = list(df.columns.values)\n",
|
|
"\n",
|
|
" # iterate as long as the value is not None\n",
|
|
" if num_rows_to_consider is None:\n",
|
|
" rows = list(df.values)\n",
|
|
" else:\n",
|
|
" rows = list(df.values)[:num_rows_to_consider]\n",
|
|
" valid_row = list(df.values[-1])\n",
|
|
" for row in rows:\n",
|
|
" for i in range(len(row)):\n",
|
|
" value = row[i]\n",
|
|
" if not pd.isnull(value):\n",
|
|
" valid_row[i] = value\n",
|
|
" last_row = valid_row\n",
|
|
" # last_row will be the row that's valid\n",
|
|
"\n",
|
|
" from collections import defaultdict\n",
|
|
" results = defaultdict(dict)\n",
|
|
" for column_title, last_row_value in zip(column_titles, last_row):\n",
|
|
" if column_title == \"Step\" or \"__MAX\" in column_title or \"__MIN\" in column_title:\n",
|
|
" continue\n",
|
|
" name = column_title.split(\" - \")[0]\n",
|
|
" capture_name, method = name.split(\"_\")\n",
|
|
" metric = float(last_row_value)\n",
|
|
" # format to have 2 decimal places\n",
|
|
" if num_decimals == 2:\n",
|
|
" metric = f\"{metric:.2f}\"\n",
|
|
" elif num_decimals == 3:\n",
|
|
" metric = f\"{metric:.3f}\"\n",
|
|
" results[capture_name][method] = metric\n",
|
|
"\n",
|
|
" # import pprint\n",
|
|
" # pprint.pprint(results)\n",
|
|
" # print(len(results))\n",
|
|
"\n",
|
|
" results = dict(results)\n",
|
|
" return results"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# the csv filenames\n",
|
|
"# these are generated from exporting CSVs from wandb graphs\n",
|
|
"# csv_filename_psrn = \"/home/ethanweber/nerfactory/wandb-psnr.csv\"\n",
|
|
"# csv_filename_ssim = \"/home/ethanweber/nerfactory/wandb-ssim.csv\"\n",
|
|
"# csv_filename_lpips = \"/home/ethanweber/nerfactory/wandb-lpips.csv\"\n",
|
|
"csv_filename_psrn = \"/home/ethanweber/nerfactory/wandb-jan9-psnr.csv\"\n",
|
|
"csv_filename_ssim = \"/home/ethanweber/nerfactory/wandb-jan9-ssim.csv\"\n",
|
|
"csv_filename_lpips = \"/home/ethanweber/nerfactory/wandb-jan9-lpips.csv\"\n",
|
|
"\n",
|
|
"results_psrn = csv_filename_to_results(csv_filename_psrn)\n",
|
|
"results_ssim = csv_filename_to_results(csv_filename_ssim, num_decimals=3)\n",
|
|
"results_lpips = csv_filename_to_results(csv_filename_lpips, num_decimals=3)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 12,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def generate_row(method, capture_names):\n",
|
|
" # remove spaces from method\n",
|
|
" method_text = method.replace(\"-\", \" \")\n",
|
|
" string = f\"{method_text}\"\n",
|
|
" for idx, capture_name in enumerate(capture_names):\n",
|
|
" psnr = results_psrn[capture_name][method]\n",
|
|
" ssim = results_ssim[capture_name][method]\n",
|
|
" lpips = results_lpips[capture_name][method]\n",
|
|
" if idx == len(capture_names) - 1:\n",
|
|
" string += \" & \\\\multicolumn{1}{r|}{\" + f\"{psnr} / {ssim} / {lpips}\" + \"}\"\n",
|
|
" else:\n",
|
|
" string += \" & \\\\multicolumn{1}{r}{\" + f\"{psnr} / {ssim} / {lpips}\" + \"}\"\n",
|
|
" return string"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 13,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"['Egypt', 'person', 'kitchen', 'plane', 'dozer']\n",
|
|
"no contraction & \\multicolumn{1}{r}{18.06 / 0.520 / 0.151} & \\multicolumn{1}{r}{21.96 / 0.511 / 0.125} & \\multicolumn{1}{r}{19.35 / 0.764 / 0.078} & \\multicolumn{1}{r}{18.45 / 0.519 / 0.161} & \\multicolumn{1}{r|}{19.53 / 0.786 / 0.081} \\\\ \n",
|
|
"synthetic on real & \\multicolumn{1}{r}{18.51 / 0.546 / 0.119} & \\multicolumn{1}{r}{24.68 / 0.667 / 0.099} & \\multicolumn{1}{r}{20.98 / 0.745 / 0.074} & \\multicolumn{1}{r}{19.32 / 0.578 / 0.102} & \\multicolumn{1}{r|}{21.41 / 0.758 / 0.080} \\\\ \\hline\n",
|
|
"\n",
|
|
"['floating-tree', 'aspen', 'stump', 'sculpture', 'Giannini-Hall']\n",
|
|
"no contraction & \\multicolumn{1}{r}{18.98 / 0.679 / 0.088} & \\multicolumn{1}{r}{15.80 / 0.330 / 0.342} & \\multicolumn{1}{r}{20.69 / 0.704 / 0.075} & \\multicolumn{1}{r}{21.43 / 0.720 / 0.073} & \\multicolumn{1}{r|}{17.75 / 0.596 / 0.134} \\\\ \n",
|
|
"synthetic on real & \\multicolumn{1}{r}{21.13 / 0.726 / 0.072} & \\multicolumn{1}{r}{15.53 / 0.301 / 0.332} & \\multicolumn{1}{r}{23.86 / 0.786 / 0.053} & \\multicolumn{1}{r}{22.62 / 0.706 / 0.076} & \\multicolumn{1}{r|}{18.41 / 0.576 / 0.133} \\\\ \\hline\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"capture_names_row1 = [\n",
|
|
" \"Egypt\",\n",
|
|
" \"person\",\n",
|
|
" \"kitchen\",\n",
|
|
" \"plane\",\n",
|
|
" \"dozer\"\n",
|
|
"]\n",
|
|
"capture_names_row2 = [\n",
|
|
" \"floating-tree\",\n",
|
|
" \"aspen\",\n",
|
|
" \"stump\",\n",
|
|
" \"sculpture\",\n",
|
|
" \"Giannini-Hall\"\n",
|
|
"]\n",
|
|
"\n",
|
|
"print(capture_names_row1)\n",
|
|
"for method in methods[:-1]:\n",
|
|
" # print(generate_row(method, capture_names_row1) + ' \\\\\\ \\cline{1-1}')\n",
|
|
" print(generate_row(method, capture_names_row1) + ' \\\\\\ ')\n",
|
|
"print(generate_row(methods[-1], capture_names_row1) + ' \\\\\\ \\hline')\n",
|
|
"\n",
|
|
"print()\n",
|
|
"\n",
|
|
"print(capture_names_row2)\n",
|
|
"for method in methods[:-1]:\n",
|
|
" # print(generate_row(method, capture_names_row2) + ' \\\\\\ \\cline{1-1}')\n",
|
|
" print(generate_row(method, capture_names_row2) + ' \\\\\\ ')\n",
|
|
"print(generate_row(methods[-1], capture_names_row2) + ' \\\\\\ \\hline')\n",
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 13,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"csv_filename_psrn = \"/home/ethanweber/nerfactory/mipnerf360-instant-ngp-psnr.csv\"\n",
|
|
"csv_filename_ssim = \"/home/ethanweber/nerfactory/mipnerf360-instant-ngp-ssim.csv\"\n",
|
|
"csv_filename_lpips = \"/home/ethanweber/nerfactory/mipnerf360-instant-ngp-lpips.csv\"\n",
|
|
"\n",
|
|
"results_psrn = csv_filename_to_results(csv_filename_psrn, num_rows_to_consider=1)\n",
|
|
"results_ssim = csv_filename_to_results(csv_filename_ssim, num_decimals=3, num_rows_to_consider=1)\n",
|
|
"results_lpips = csv_filename_to_results(csv_filename_lpips, num_decimals=3, num_rows_to_consider=1)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"psnr\n",
|
|
"[('bicycle', {'instant-ngp-w/o-pose-app': '19.92'}),\n",
|
|
" ('garden', {'instant-ngp-w/o-pose-app': '22.68'}),\n",
|
|
" ('stump', {'instant-ngp-w/o-pose-app': '20.25'}),\n",
|
|
" ('room', {'instant-ngp-w/o-pose-app': '28.04'}),\n",
|
|
" ('counter', {'instant-ngp-w/o-pose-app': '23.59'}),\n",
|
|
" ('kitchen', {'instant-ngp-w/o-pose-app': '24.61'}),\n",
|
|
" ('bonsai', {'instant-ngp-w/o-pose-app': '25.67'})]\n",
|
|
"\n",
|
|
"ssim\n",
|
|
"[('bicycle', {'instant-ngp-w/o-pose-app': '0.448'}),\n",
|
|
" ('garden', {'instant-ngp-w/o-pose-app': '0.542'}),\n",
|
|
" ('stump', {'instant-ngp-w/o-pose-app': '0.465'}),\n",
|
|
" ('room', {'instant-ngp-w/o-pose-app': '0.828'}),\n",
|
|
" ('counter', {'instant-ngp-w/o-pose-app': '0.701'}),\n",
|
|
" ('kitchen', {'instant-ngp-w/o-pose-app': '0.642'}),\n",
|
|
" ('bonsai', {'instant-ngp-w/o-pose-app': '0.792'})]\n",
|
|
"\n",
|
|
"lpips\n",
|
|
"[('bicycle', {'instant-ngp-w/o-pose-app': '0.449'}),\n",
|
|
" ('garden', {'instant-ngp-w/o-pose-app': '0.402'}),\n",
|
|
" ('stump', {'instant-ngp-w/o-pose-app': '0.440'}),\n",
|
|
" ('room', {'instant-ngp-w/o-pose-app': '0.170'}),\n",
|
|
" ('counter', {'instant-ngp-w/o-pose-app': '0.271'}),\n",
|
|
" ('kitchen', {'instant-ngp-w/o-pose-app': '0.281'}),\n",
|
|
" ('bonsai', {'instant-ngp-w/o-pose-app': '0.186'})]\n",
|
|
"\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"mipnerf360_nerfacto_capture_names = [\"bicycle\", \"garden\", \"stump\", \"room\", \"counter\", \"kitchen\", \"bonsai\"]\n",
|
|
"\n",
|
|
"print(\"psnr\")\n",
|
|
"s = [(key, results_psrn[key]) for key in mipnerf360_nerfacto_capture_names]\n",
|
|
"pprint.pprint(s)\n",
|
|
"print()\n",
|
|
"\n",
|
|
"print(\"ssim\")\n",
|
|
"s = [(key, results_ssim[key]) for key in mipnerf360_nerfacto_capture_names]\n",
|
|
"pprint.pprint(s)\n",
|
|
"print()\n",
|
|
"\n",
|
|
"print(\"lpips\")\n",
|
|
"s = [(key, results_lpips[key]) for key in mipnerf360_nerfacto_capture_names]\n",
|
|
"pprint.pprint(s)\n",
|
|
"print()\n",
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 14,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"psnr\n",
|
|
"[('bicycle', {'instant-ngp-w/o-pose-app': '18.78'}),\n",
|
|
" ('garden', {'instant-ngp-w/o-pose-app': '20.82'}),\n",
|
|
" ('stump', {'instant-ngp-w/o-pose-app': '19.73'}),\n",
|
|
" ('room', {'instant-ngp-w/o-pose-app': '25.26'}),\n",
|
|
" ('counter', {'instant-ngp-w/o-pose-app': '22.09'}),\n",
|
|
" ('kitchen', {'instant-ngp-w/o-pose-app': '22.82'}),\n",
|
|
" ('bonsai', {'instant-ngp-w/o-pose-app': '23.86'})]\n",
|
|
"\n",
|
|
"ssim\n",
|
|
"[('bicycle', {'instant-ngp-w/o-pose-app': '0.389'}),\n",
|
|
" ('garden', {'instant-ngp-w/o-pose-app': '0.390'}),\n",
|
|
" ('stump', {'instant-ngp-w/o-pose-app': '0.408'}),\n",
|
|
" ('room', {'instant-ngp-w/o-pose-app': '0.749'}),\n",
|
|
" ('counter', {'instant-ngp-w/o-pose-app': '0.621'}),\n",
|
|
" ('kitchen', {'instant-ngp-w/o-pose-app': '0.530'}),\n",
|
|
" ('bonsai', {'instant-ngp-w/o-pose-app': '0.665'})]\n",
|
|
"\n",
|
|
"lpips\n",
|
|
"[('bicycle', {'instant-ngp-w/o-pose-app': '0.620'}),\n",
|
|
" ('garden', {'instant-ngp-w/o-pose-app': '0.619'}),\n",
|
|
" ('stump', {'instant-ngp-w/o-pose-app': '0.607'}),\n",
|
|
" ('room', {'instant-ngp-w/o-pose-app': '0.311'}),\n",
|
|
" ('counter', {'instant-ngp-w/o-pose-app': '0.417'}),\n",
|
|
" ('kitchen', {'instant-ngp-w/o-pose-app': '0.427'}),\n",
|
|
" ('bonsai', {'instant-ngp-w/o-pose-app': '0.382'})]\n",
|
|
"\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"mipnerf360_nerfacto_capture_names = [\"bicycle\", \"garden\", \"stump\", \"room\", \"counter\", \"kitchen\", \"bonsai\"]\n",
|
|
"\n",
|
|
"print(\"psnr\")\n",
|
|
"s = [(key, results_psrn[key]) for key in mipnerf360_nerfacto_capture_names]\n",
|
|
"pprint.pprint(s)\n",
|
|
"print()\n",
|
|
"\n",
|
|
"print(\"ssim\")\n",
|
|
"s = [(key, results_ssim[key]) for key in mipnerf360_nerfacto_capture_names]\n",
|
|
"pprint.pprint(s)\n",
|
|
"print()\n",
|
|
"\n",
|
|
"print(\"lpips\")\n",
|
|
"s = [(key, results_lpips[key]) for key in mipnerf360_nerfacto_capture_names]\n",
|
|
"pprint.pprint(s)\n",
|
|
"print()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "nerfactory",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.10.6"
|
|
},
|
|
"orig_nbformat": 4,
|
|
"vscode": {
|
|
"interpreter": {
|
|
"hash": "b7b86c17c0dfd6b313529972d424f2e075cc198a46cda005757126afe8133c7b"
|
|
}
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|