mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-11 18:44:21 +08:00
nerfacto ablations table code
This commit is contained in:
Ethan Weber
parent
0ff9b29538
commit
d719760970
3
projects/nerfstudio_paper/.gitignore
vendored
3
projects/nerfstudio_paper/.gitignore
vendored
@ -1,3 +1,4 @@
|
||||
camera_paths/
|
||||
notes/
|
||||
renders/
|
||||
renders/
|
||||
wandb/
|
||||
264
projects/nerfstudio_paper/create_tables.ipynb
Normal file
264
projects/nerfstudio_paper/create_tables.ipynb
Normal file
@ -0,0 +1,264 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%load_ext autoreload\n",
|
||||
"%autoreload 2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"['nerfacto', 'w/o-pose', 'w/o-app', 'w/o-pose-app', '1-prop-network', 'l2-contraction', 'shared-prop-network', 'random-background-color', 'no-contraction', 'synthetic-on-real']\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import copy\n",
|
||||
"import pprint\n",
|
||||
"from benchmark_nerfstudio_paper import ablations_table_rows\n",
|
||||
"from collections import defaultdict\n",
|
||||
"\n",
|
||||
"# methods sorted in the same order we want in the paper tables\n",
|
||||
"nerfacto_ablations_methods = [row[0] for row in ablations_table_rows]\n",
|
||||
"print(nerfacto_ablations_methods)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def csv_filename_to_results(csv_filename, num_decimals=2, num_rows_to_consider=None):\n",
|
||||
" # code to read the csv file\n",
|
||||
" # and return a dictionary\n",
|
||||
"\n",
|
||||
" df = pd.read_csv(csv_filename)\n",
|
||||
"\n",
|
||||
" column_titles = list(df.columns.values)\n",
|
||||
"\n",
|
||||
" # iterate as long as the value is not None\n",
|
||||
" if num_rows_to_consider is None:\n",
|
||||
" rows = list(df.values)\n",
|
||||
" else:\n",
|
||||
" rows = list(df.values)[:num_rows_to_consider]\n",
|
||||
" valid_row = list(df.values[-1])\n",
|
||||
" for row in rows:\n",
|
||||
" for i in range(len(row)):\n",
|
||||
" value = row[i]\n",
|
||||
" if not pd.isnull(value):\n",
|
||||
" valid_row[i] = value\n",
|
||||
" last_row = valid_row\n",
|
||||
" # last_row will be the row that's valid\n",
|
||||
"\n",
|
||||
" results = defaultdict(dict)\n",
|
||||
" for column_title, last_row_value in zip(column_titles, last_row):\n",
|
||||
" if column_title == \"Step\" or \"__MAX\" in column_title or \"__MIN\" in column_title:\n",
|
||||
" continue\n",
|
||||
" name = column_title.split(\" - \")[0]\n",
|
||||
" capture_name, method = name.split(\"_\")\n",
|
||||
" metric = float(last_row_value)\n",
|
||||
" # format to have 2 decimal places\n",
|
||||
" if num_decimals == 2:\n",
|
||||
" metric = f\"{metric:.2f}\"\n",
|
||||
" elif num_decimals == 3:\n",
|
||||
" metric = f\"{metric:.3f}\"\n",
|
||||
" else:\n",
|
||||
" metric = f\"{metric}\"\n",
|
||||
" results[capture_name][method] = metric\n",
|
||||
"\n",
|
||||
" results = dict(results)\n",
|
||||
" return results"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"csv_filename_psnr = \"./wandb/nerfacto_ablations_psnr.csv\"\n",
|
||||
"csv_filename_ssim = \"./wandb/nerfacto_ablations_ssim.csv\"\n",
|
||||
"csv_filename_lpips = \"./wandb/nerfacto_ablations_lpips.csv\"\n",
|
||||
"results_psnr = csv_filename_to_results(csv_filename_psnr, num_decimals=2)\n",
|
||||
"results_ssim = csv_filename_to_results(csv_filename_ssim, num_decimals=3)\n",
|
||||
"results_lpips = csv_filename_to_results(csv_filename_lpips, num_decimals=3)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def generate_row(method, capture_names):\n",
|
||||
" # remove spaces from method\n",
|
||||
" method_text = method.replace(\"-\", \" \")\n",
|
||||
" string = f\"{method_text}\"\n",
|
||||
" for idx, capture_name in enumerate(capture_names):\n",
|
||||
" psnr = results_psnr[capture_name][method]\n",
|
||||
" ssim = results_ssim[capture_name][method]\n",
|
||||
" lpips = results_lpips[capture_name][method]\n",
|
||||
" if idx == len(capture_names) - 1:\n",
|
||||
" string += \" & \\\\multicolumn{1}{r|}{\" + f\"{psnr} / {ssim} / {lpips}\" + \"}\"\n",
|
||||
" else:\n",
|
||||
" string += \" & \\\\multicolumn{1}{r}{\" + f\"{psnr} / {ssim} / {lpips}\" + \"}\"\n",
|
||||
" return string"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"['Egypt', 'person', 'kitchen', 'plane', 'dozer']\n",
|
||||
"nerfacto & \\multicolumn{1}{r}{21.67 / 0.689 / 0.375} & \\multicolumn{1}{r}{25.17 / 0.692 / 0.320} & \\multicolumn{1}{r}{20.55 / 0.807 / 0.389} & \\multicolumn{1}{r}{22.11 / 0.649 / 0.419} & \\multicolumn{1}{r|}{20.20 / 0.743 / 0.391} \\\\ \n",
|
||||
"w/o pose & \\multicolumn{1}{r}{21.56 / 0.689 / 0.374} & \\multicolumn{1}{r}{25.52 / 0.716 / 0.323} & \\multicolumn{1}{r}{20.47 / 0.798 / 0.402} & \\multicolumn{1}{r}{21.83 / 0.645 / 0.421} & \\multicolumn{1}{r|}{20.38 / 0.747 / 0.390} \\\\ \n",
|
||||
"w/o app & \\multicolumn{1}{r}{22.38 / 0.689 / 0.396} & \\multicolumn{1}{r}{27.74 / 0.744 / 0.316} & \\multicolumn{1}{r}{23.80 / 0.819 / 0.397} & \\multicolumn{1}{r}{22.44 / 0.641 / 0.440} & \\multicolumn{1}{r|}{23.58 / 0.750 / 0.393} \\\\ \n",
|
||||
"w/o pose app & \\multicolumn{1}{r}{22.31 / 0.692 / 0.399} & \\multicolumn{1}{r}{27.76 / 0.749 / 0.328} & \\multicolumn{1}{r}{23.39 / 0.808 / 0.409} & \\multicolumn{1}{r}{22.22 / 0.642 / 0.443} & \\multicolumn{1}{r|}{23.38 / 0.748 / 0.394} \\\\ \n",
|
||||
"1 prop network & \\multicolumn{1}{r}{21.65 / 0.682 / 0.387} & \\multicolumn{1}{r}{25.80 / 0.730 / 0.324} & \\multicolumn{1}{r}{20.57 / 0.805 / 0.394} & \\multicolumn{1}{r}{22.08 / 0.648 / 0.422} & \\multicolumn{1}{r|}{20.00 / 0.738 / 0.403} \\\\ \n",
|
||||
"l2 contraction & \\multicolumn{1}{r}{21.71 / 0.691 / 0.375} & \\multicolumn{1}{r}{25.05 / 0.690 / 0.322} & \\multicolumn{1}{r}{20.62 / 0.808 / 0.386} & \\multicolumn{1}{r}{22.07 / 0.649 / 0.416} & \\multicolumn{1}{r|}{20.47 / 0.745 / 0.386} \\\\ \n",
|
||||
"shared prop network & \\multicolumn{1}{r}{21.64 / 0.687 / 0.382} & \\multicolumn{1}{r}{25.11 / 0.691 / 0.322} & \\multicolumn{1}{r}{20.56 / 0.805 / 0.392} & \\multicolumn{1}{r}{22.11 / 0.650 / 0.416} & \\multicolumn{1}{r|}{20.46 / 0.746 / 0.385} \\\\ \n",
|
||||
"random background color & \\multicolumn{1}{r}{21.56 / 0.678 / 0.389} & \\multicolumn{1}{r}{25.54 / 0.709 / 0.328} & \\multicolumn{1}{r}{20.64 / 0.809 / 0.390} & \\multicolumn{1}{r}{21.95 / 0.642 / 0.423} & \\multicolumn{1}{r|}{20.29 / 0.746 / 0.387} \\\\ \n",
|
||||
"no contraction & \\multicolumn{1}{r}{17.53 / 0.467 / 0.585} & \\multicolumn{1}{r}{21.64 / 0.515 / 0.511} & \\multicolumn{1}{r}{19.12 / 0.724 / 0.440} & \\multicolumn{1}{r}{17.78 / 0.492 / 0.582} & \\multicolumn{1}{r|}{18.97 / 0.659 / 0.433} \\\\ \n",
|
||||
"synthetic on real & \\multicolumn{1}{r}{18.07 / 0.471 / 0.574} & \\multicolumn{1}{r}{23.68 / 0.555 / 0.494} & \\multicolumn{1}{r}{21.21 / 0.727 / 0.453} & \\multicolumn{1}{r}{18.37 / 0.511 / 0.553} & \\multicolumn{1}{r|}{21.17 / 0.657 / 0.441} \\\\ \\hline\n",
|
||||
"\n",
|
||||
"['floating-tree', 'aspen', 'stump', 'sculpture', 'Giannini-Hall']\n",
|
||||
"nerfacto & \\multicolumn{1}{r}{20.03 / 0.740 / 0.312} & \\multicolumn{1}{r}{16.95 / 0.347 / 0.560} & \\multicolumn{1}{r}{22.04 / 0.698 / 0.295} & \\multicolumn{1}{r}{21.88 / 0.676 / 0.368} & \\multicolumn{1}{r|}{19.26 / 0.593 / 0.462} \\\\ \n",
|
||||
"w/o pose & \\multicolumn{1}{r}{19.80 / 0.721 / 0.321} & \\multicolumn{1}{r}{16.96 / 0.345 / 0.560} & \\multicolumn{1}{r}{21.65 / 0.667 / 0.305} & \\multicolumn{1}{r}{21.82 / 0.670 / 0.368} & \\multicolumn{1}{r|}{19.28 / 0.591 / 0.461} \\\\ \n",
|
||||
"w/o app & \\multicolumn{1}{r}{21.84 / 0.727 / 0.329} & \\multicolumn{1}{r}{16.92 / 0.333 / 0.567} & \\multicolumn{1}{r}{25.11 / 0.777 / 0.308} & \\multicolumn{1}{r}{22.16 / 0.657 / 0.422} & \\multicolumn{1}{r|}{20.54 / 0.587 / 0.495} \\\\ \n",
|
||||
"w/o pose app & \\multicolumn{1}{r}{21.77 / 0.719 / 0.341} & \\multicolumn{1}{r}{16.91 / 0.334 / 0.567} & \\multicolumn{1}{r}{24.87 / 0.778 / 0.307} & \\multicolumn{1}{r}{22.08 / 0.650 / 0.428} & \\multicolumn{1}{r|}{20.63 / 0.587 / 0.494} \\\\ \n",
|
||||
"1 prop network & \\multicolumn{1}{r}{19.96 / 0.736 / 0.318} & \\multicolumn{1}{r}{16.79 / 0.325 / 0.587} & \\multicolumn{1}{r}{22.70 / 0.751 / 0.283} & \\multicolumn{1}{r}{21.99 / 0.685 / 0.365} & \\multicolumn{1}{r|}{19.20 / 0.585 / 0.475} \\\\ \n",
|
||||
"l2 contraction & \\multicolumn{1}{r}{19.81 / 0.741 / 0.311} & \\multicolumn{1}{r}{16.89 / 0.346 / 0.562} & \\multicolumn{1}{r}{22.16 / 0.708 / 0.296} & \\multicolumn{1}{r}{21.84 / 0.673 / 0.369} & \\multicolumn{1}{r|}{19.21 / 0.592 / 0.457} \\\\ \n",
|
||||
"shared prop network & \\multicolumn{1}{r}{19.91 / 0.734 / 0.315} & \\multicolumn{1}{r}{16.95 / 0.344 / 0.561} & \\multicolumn{1}{r}{21.73 / 0.692 / 0.301} & \\multicolumn{1}{r}{21.75 / 0.670 / 0.374} & \\multicolumn{1}{r|}{19.25 / 0.590 / 0.462} \\\\ \n",
|
||||
"random background color & \\multicolumn{1}{r}{19.85 / 0.729 / 0.316} & \\multicolumn{1}{r}{16.95 / 0.343 / 0.561} & \\multicolumn{1}{r}{22.20 / 0.711 / 0.294} & \\multicolumn{1}{r}{21.84 / 0.677 / 0.370} & \\multicolumn{1}{r|}{19.23 / 0.590 / 0.457} \\\\ \n",
|
||||
"no contraction & \\multicolumn{1}{r}{18.85 / 0.588 / 0.410} & \\multicolumn{1}{r}{14.13 / 0.170 / 0.717} & \\multicolumn{1}{r}{20.27 / 0.630 / 0.392} & \\multicolumn{1}{r}{20.71 / 0.619 / 0.436} & \\multicolumn{1}{r|}{16.92 / 0.472 / 0.557} \\\\ \n",
|
||||
"synthetic on real & \\multicolumn{1}{r}{20.64 / 0.565 / 0.418} & \\multicolumn{1}{r}{14.76 / 0.186 / 0.690} & \\multicolumn{1}{r}{23.56 / 0.675 / 0.400} & \\multicolumn{1}{r}{21.71 / 0.609 / 0.485} & \\multicolumn{1}{r|}{17.74 / 0.468 / 0.579} \\\\ \\hline\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"capture_names_row1 = [\"Egypt\", \"person\", \"kitchen\", \"plane\", \"dozer\"]\n",
|
||||
"capture_names_row2 = [\"floating-tree\", \"aspen\", \"stump\", \"sculpture\", \"Giannini-Hall\"]\n",
|
||||
"\n",
|
||||
"print(capture_names_row1)\n",
|
||||
"for method in nerfacto_ablations_methods[:-1]:\n",
|
||||
" print(generate_row(method, capture_names_row1) + \" \\\\\\ \")\n",
|
||||
"print(generate_row(nerfacto_ablations_methods[-1], capture_names_row1) + \" \\\\\\ \\hline\")\n",
|
||||
"\n",
|
||||
"print()\n",
|
||||
"\n",
|
||||
"print(capture_names_row2)\n",
|
||||
"for method in nerfacto_ablations_methods[:-1]:\n",
|
||||
" print(generate_row(method, capture_names_row2) + \" \\\\\\ \")\n",
|
||||
"print(generate_row(nerfacto_ablations_methods[-1], capture_names_row2) + \" \\\\\\ \\hline\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"nerfacto & 20.99 & 0.663 & 0.389 \\\\\n",
|
||||
"w/o-pose & 20.93 & 0.659 & 0.393 \\\\\n",
|
||||
"w/o-app & 22.65 & 0.672 & 0.406 \\\\\n",
|
||||
"w/o-pose-app & 22.53 & 0.671 & 0.411 \\\\\n",
|
||||
"1-prop-network & 21.07 & 0.669 & 0.396 \\\\\n",
|
||||
"l2-contraction & 20.98 & 0.664 & 0.388 \\\\\n",
|
||||
"shared-prop-network & 20.95 & 0.661 & 0.391 \\\\\n",
|
||||
"random-background-color & 21.0 & 0.663 & 0.392 \\\\\n",
|
||||
"no-contraction & 18.59 & 0.534 & 0.506 \\\\\n",
|
||||
"synthetic-on-real & 20.09 & 0.542 & 0.509 \\\\\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"def get_averages_from_results(results, num_decimals=2):\n",
|
||||
" averages = defaultdict(list)\n",
|
||||
" for capture_name, methods in results.items():\n",
|
||||
" for method, metric in methods.items():\n",
|
||||
" averages[method].append(float(metric))\n",
|
||||
" for method, metrics in averages.items():\n",
|
||||
" val = sum(metrics) / len(metrics)\n",
|
||||
" if num_decimals == 2:\n",
|
||||
" metric = f\"{val:.2f}\"\n",
|
||||
" elif num_decimals == 3:\n",
|
||||
" metric = f\"{val:.3f}\"\n",
|
||||
" else:\n",
|
||||
" metric = f\"{val}\"\n",
|
||||
" averages[method] = float(metric)\n",
|
||||
" \n",
|
||||
" return averages\n",
|
||||
"\n",
|
||||
"results_average_psnr = get_averages_from_results(results_psnr, num_decimals=2)\n",
|
||||
"results_average_ssim = get_averages_from_results(results_ssim, num_decimals=3)\n",
|
||||
"results_average_lpips = get_averages_from_results(results_lpips, num_decimals=3)\n",
|
||||
"\n",
|
||||
"for method in nerfacto_ablations_methods:\n",
|
||||
" row = f\"{method} & {results_average_psnr[method]} & {results_average_ssim[method]} & {results_average_lpips[method]} \\\\\\\\\"\n",
|
||||
" print(row)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "nerfstudio",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.8.16"
|
||||
},
|
||||
"orig_nbformat": 4,
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "aa8854f1181fba3d1e5e00a1960f28acb4283cf16bfa87e419142a9fccb372ca"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user