nerfstudio/projects/nerfstudio_paper/create_tables.ipynb

In [1]:

%load_ext autoreload
%autoreload 2

In [2]:

import pandas as pd
import copy
import pprint
from benchmark_nerfstudio_paper import ablations_table_rows
from collections import defaultdict

# methods sorted in the same order we want in the paper tables
nerfacto_ablations_methods = [row[0] for row in ablations_table_rows]
print(nerfacto_ablations_methods)

['nerfacto', 'w/o-pose', 'w/o-app', 'w/o-pose-app', '1-prop-network', 'l2-contraction', 'shared-prop-network', 'random-background-color', 'no-contraction', 'synthetic-on-real']

In [3]:

def csv_filename_to_results(csv_filename, num_decimals=2, num_rows_to_consider=None):
    # code to read the csv file
    # and return a dictionary

    df = pd.read_csv(csv_filename)

    column_titles = list(df.columns.values)

    # iterate as long as the value is not None
    if num_rows_to_consider is None:
        rows = list(df.values)
    else:
        rows = list(df.values)[:num_rows_to_consider]
    valid_row = list(df.values[-1])
    for row in rows:
        for i in range(len(row)):
            value = row[i]
            if not pd.isnull(value):
                valid_row[i] = value
    last_row = valid_row
    # last_row will be the row that's valid

    results = defaultdict(dict)
    for column_title, last_row_value in zip(column_titles, last_row):
        if column_title == "Step" or "__MAX" in column_title or "__MIN" in column_title:
            continue
        name = column_title.split(" - ")[0]
        capture_name, method = name.split("_")
        metric = float(last_row_value)
        # format to have 2 decimal places
        if num_decimals == 2:
            metric = f"{metric:.2f}"
        elif num_decimals == 3:
            metric = f"{metric:.3f}"
        else:
            metric = f"{metric}"
        results[capture_name][method] = metric

    results = dict(results)
    return results

In [4]:

csv_filename_psnr = "./wandb/nerfacto_ablations_psnr.csv"
csv_filename_ssim = "./wandb/nerfacto_ablations_ssim.csv"
csv_filename_lpips = "./wandb/nerfacto_ablations_lpips.csv"
results_psnr = csv_filename_to_results(csv_filename_psnr, num_decimals=2)
results_ssim = csv_filename_to_results(csv_filename_ssim, num_decimals=3)
results_lpips = csv_filename_to_results(csv_filename_lpips, num_decimals=3)

In [5]:

def generate_row(method, capture_names):
    # remove spaces from method
    method_text = method.replace("-", " ")
    string = f"{method_text}"
    for idx, capture_name in enumerate(capture_names):
        psnr = results_psnr[capture_name][method]
        ssim = results_ssim[capture_name][method]
        lpips = results_lpips[capture_name][method]
        if idx == len(capture_names) - 1:
            string += " & \\multicolumn{1}{r|}{" + f"{psnr} / {ssim} / {lpips}" + "}"
        else:
            string += " & \\multicolumn{1}{r}{" + f"{psnr} / {ssim} / {lpips}" + "}"
    return string

In [6]:

capture_names_row1 = ["Egypt", "person", "kitchen", "plane", "dozer"]
capture_names_row2 = ["floating-tree", "aspen", "stump", "sculpture", "Giannini-Hall"]

print(capture_names_row1)
for method in nerfacto_ablations_methods[:-1]:
    print(generate_row(method, capture_names_row1) + " \\\ ")
print(generate_row(nerfacto_ablations_methods[-1], capture_names_row1) + " \\\ \hline")

print()

print(capture_names_row2)
for method in nerfacto_ablations_methods[:-1]:
    print(generate_row(method, capture_names_row2) + " \\\ ")
print(generate_row(nerfacto_ablations_methods[-1], capture_names_row2) + " \\\ \hline")

['Egypt', 'person', 'kitchen', 'plane', 'dozer']
nerfacto & \multicolumn{1}{r}{21.67 / 0.689 / 0.375} & \multicolumn{1}{r}{25.17 / 0.692 / 0.320} & \multicolumn{1}{r}{20.55 / 0.807 / 0.389} & \multicolumn{1}{r}{22.11 / 0.649 / 0.419} & \multicolumn{1}{r|}{20.20 / 0.743 / 0.391} \\ 
w/o pose & \multicolumn{1}{r}{21.56 / 0.689 / 0.374} & \multicolumn{1}{r}{25.52 / 0.716 / 0.323} & \multicolumn{1}{r}{20.47 / 0.798 / 0.402} & \multicolumn{1}{r}{21.83 / 0.645 / 0.421} & \multicolumn{1}{r|}{20.38 / 0.747 / 0.390} \\ 
w/o app & \multicolumn{1}{r}{22.38 / 0.689 / 0.396} & \multicolumn{1}{r}{27.74 / 0.744 / 0.316} & \multicolumn{1}{r}{23.80 / 0.819 / 0.397} & \multicolumn{1}{r}{22.44 / 0.641 / 0.440} & \multicolumn{1}{r|}{23.58 / 0.750 / 0.393} \\ 
w/o pose app & \multicolumn{1}{r}{22.31 / 0.692 / 0.399} & \multicolumn{1}{r}{27.76 / 0.749 / 0.328} & \multicolumn{1}{r}{23.39 / 0.808 / 0.409} & \multicolumn{1}{r}{22.22 / 0.642 / 0.443} & \multicolumn{1}{r|}{23.38 / 0.748 / 0.394} \\ 
1 prop network & \multicolumn{1}{r}{21.65 / 0.682 / 0.387} & \multicolumn{1}{r}{25.80 / 0.730 / 0.324} & \multicolumn{1}{r}{20.57 / 0.805 / 0.394} & \multicolumn{1}{r}{22.08 / 0.648 / 0.422} & \multicolumn{1}{r|}{20.00 / 0.738 / 0.403} \\ 
l2 contraction & \multicolumn{1}{r}{21.71 / 0.691 / 0.375} & \multicolumn{1}{r}{25.05 / 0.690 / 0.322} & \multicolumn{1}{r}{20.62 / 0.808 / 0.386} & \multicolumn{1}{r}{22.07 / 0.649 / 0.416} & \multicolumn{1}{r|}{20.47 / 0.745 / 0.386} \\ 
shared prop network & \multicolumn{1}{r}{21.64 / 0.687 / 0.382} & \multicolumn{1}{r}{25.11 / 0.691 / 0.322} & \multicolumn{1}{r}{20.56 / 0.805 / 0.392} & \multicolumn{1}{r}{22.11 / 0.650 / 0.416} & \multicolumn{1}{r|}{20.46 / 0.746 / 0.385} \\ 
random background color & \multicolumn{1}{r}{21.56 / 0.678 / 0.389} & \multicolumn{1}{r}{25.54 / 0.709 / 0.328} & \multicolumn{1}{r}{20.64 / 0.809 / 0.390} & \multicolumn{1}{r}{21.95 / 0.642 / 0.423} & \multicolumn{1}{r|}{20.29 / 0.746 / 0.387} \\ 
no contraction & \multicolumn{1}{r}{17.53 / 0.467 / 0.585} & \multicolumn{1}{r}{21.64 / 0.515 / 0.511} & \multicolumn{1}{r}{19.12 / 0.724 / 0.440} & \multicolumn{1}{r}{17.78 / 0.492 / 0.582} & \multicolumn{1}{r|}{18.97 / 0.659 / 0.433} \\ 
synthetic on real & \multicolumn{1}{r}{18.07 / 0.471 / 0.574} & \multicolumn{1}{r}{23.68 / 0.555 / 0.494} & \multicolumn{1}{r}{21.21 / 0.727 / 0.453} & \multicolumn{1}{r}{18.37 / 0.511 / 0.553} & \multicolumn{1}{r|}{21.17 / 0.657 / 0.441} \\ \hline

['floating-tree', 'aspen', 'stump', 'sculpture', 'Giannini-Hall']
nerfacto & \multicolumn{1}{r}{20.03 / 0.740 / 0.312} & \multicolumn{1}{r}{16.95 / 0.347 / 0.560} & \multicolumn{1}{r}{22.04 / 0.698 / 0.295} & \multicolumn{1}{r}{21.88 / 0.676 / 0.368} & \multicolumn{1}{r|}{19.26 / 0.593 / 0.462} \\ 
w/o pose & \multicolumn{1}{r}{19.80 / 0.721 / 0.321} & \multicolumn{1}{r}{16.96 / 0.345 / 0.560} & \multicolumn{1}{r}{21.65 / 0.667 / 0.305} & \multicolumn{1}{r}{21.82 / 0.670 / 0.368} & \multicolumn{1}{r|}{19.28 / 0.591 / 0.461} \\ 
w/o app & \multicolumn{1}{r}{21.84 / 0.727 / 0.329} & \multicolumn{1}{r}{16.92 / 0.333 / 0.567} & \multicolumn{1}{r}{25.11 / 0.777 / 0.308} & \multicolumn{1}{r}{22.16 / 0.657 / 0.422} & \multicolumn{1}{r|}{20.54 / 0.587 / 0.495} \\ 
w/o pose app & \multicolumn{1}{r}{21.77 / 0.719 / 0.341} & \multicolumn{1}{r}{16.91 / 0.334 / 0.567} & \multicolumn{1}{r}{24.87 / 0.778 / 0.307} & \multicolumn{1}{r}{22.08 / 0.650 / 0.428} & \multicolumn{1}{r|}{20.63 / 0.587 / 0.494} \\ 
1 prop network & \multicolumn{1}{r}{19.96 / 0.736 / 0.318} & \multicolumn{1}{r}{16.79 / 0.325 / 0.587} & \multicolumn{1}{r}{22.70 / 0.751 / 0.283} & \multicolumn{1}{r}{21.99 / 0.685 / 0.365} & \multicolumn{1}{r|}{19.20 / 0.585 / 0.475} \\ 
l2 contraction & \multicolumn{1}{r}{19.81 / 0.741 / 0.311} & \multicolumn{1}{r}{16.89 / 0.346 / 0.562} & \multicolumn{1}{r}{22.16 / 0.708 / 0.296} & \multicolumn{1}{r}{21.84 / 0.673 / 0.369} & \multicolumn{1}{r|}{19.21 / 0.592 / 0.457} \\ 
shared prop network & \multicolumn{1}{r}{19.91 / 0.734 / 0.315} & \multicolumn{1}{r}{16.95 / 0.344 / 0.561} & \multicolumn{1}{r}{21.73 / 0.692 / 0.301} & \multicolumn{1}{r}{21.75 / 0.670 / 0.374} & \multicolumn{1}{r|}{19.25 / 0.590 / 0.462} \\ 
random background color & \multicolumn{1}{r}{19.85 / 0.729 / 0.316} & \multicolumn{1}{r}{16.95 / 0.343 / 0.561} & \multicolumn{1}{r}{22.20 / 0.711 / 0.294} & \multicolumn{1}{r}{21.84 / 0.677 / 0.370} & \multicolumn{1}{r|}{19.23 / 0.590 / 0.457} \\ 
no contraction & \multicolumn{1}{r}{18.85 / 0.588 / 0.410} & \multicolumn{1}{r}{14.13 / 0.170 / 0.717} & \multicolumn{1}{r}{20.27 / 0.630 / 0.392} & \multicolumn{1}{r}{20.71 / 0.619 / 0.436} & \multicolumn{1}{r|}{16.92 / 0.472 / 0.557} \\ 
synthetic on real & \multicolumn{1}{r}{20.64 / 0.565 / 0.418} & \multicolumn{1}{r}{14.76 / 0.186 / 0.690} & \multicolumn{1}{r}{23.56 / 0.675 / 0.400} & \multicolumn{1}{r}{21.71 / 0.609 / 0.485} & \multicolumn{1}{r|}{17.74 / 0.468 / 0.579} \\ \hline

In [7]:

def get_averages_from_results(results, num_decimals=2):
    averages = defaultdict(list)
    for capture_name, methods in results.items():
        for method, metric in methods.items():
            averages[method].append(float(metric))
    for method, metrics in averages.items():
        val = sum(metrics) / len(metrics)
        if num_decimals == 2:
            metric = f"{val:.2f}"
        elif num_decimals == 3:
            metric = f"{val:.3f}"
        else:
            metric = f"{val}"
        averages[method] = float(metric)
        
    return averages

results_average_psnr = get_averages_from_results(results_psnr, num_decimals=2)
results_average_ssim = get_averages_from_results(results_ssim, num_decimals=3)
results_average_lpips = get_averages_from_results(results_lpips, num_decimals=3)

for method in nerfacto_ablations_methods:
    row = f"{method} & {results_average_psnr[method]} & {results_average_ssim[method]} & {results_average_lpips[method]} \\\\"
    print(row)

nerfacto & 20.99 & 0.663 & 0.389 \\
w/o-pose & 20.93 & 0.659 & 0.393 \\
w/o-app & 22.65 & 0.672 & 0.406 \\
w/o-pose-app & 22.53 & 0.671 & 0.411 \\
1-prop-network & 21.07 & 0.669 & 0.396 \\
l2-contraction & 20.98 & 0.664 & 0.388 \\
shared-prop-network & 20.95 & 0.661 & 0.391 \\
random-background-color & 21.0 & 0.663 & 0.392 \\
no-contraction & 18.59 & 0.534 & 0.506 \\
synthetic-on-real & 20.09 & 0.542 & 0.509 \\

In [ ]:

In [ ]: