nerfstudio/scripts/benchmarking/create_tables.ipynb

In [2]:

%load_ext autoreload
%autoreload 2

In [3]:

import pandas as pd
import copy
from benchmark_nerfacto import table_rows

# methods sorted in the same order we want in the paper tables
methods = [row[0] for row in table_rows]

In [4]:

def csv_filename_to_results(csv_filename, num_decimals=2):
    # code to read the csv file
    # and return a dictionary

    df = pd.read_csv(csv_filename)

    column_titles = list(df.columns.values)
    last_row = list(df.values[-1])

    # print(len(column_titles))
    # print(len(last_row))

    from collections import defaultdict
    results = defaultdict(dict)
    for column_title, last_row_value in zip(column_titles, last_row):
        if column_title == "Step" or "__MAX" in column_title or "__MIN" in column_title:
            continue
        name = column_title.split(" - ")[0]
        capture_name, method = name.split("_")
        metric = float(last_row_value)
        # format to have 2 decimal places
        if num_decimals == 2:
            metric = f"{metric:.2f}"
        elif num_decimals == 3:
            metric = f"{metric:.3f}"
        results[capture_name][method] = metric

    # import pprint
    # pprint.pprint(results)
    # print(len(results))

    results = dict(results)
    return results

In [11]:

# the csv filenames
# these are generated from exporting CSVs from wandb graphs
# csv_filename_psrn = "/home/ethanweber/nerfactory/wandb-psnr.csv"
# csv_filename_ssim = "/home/ethanweber/nerfactory/wandb-ssim.csv"
# csv_filename_lpips = "/home/ethanweber/nerfactory/wandb-lpips.csv"
csv_filename_psrn = "/home/ethanweber/nerfactory/wandb-jan9-psnr.csv"
csv_filename_ssim = "/home/ethanweber/nerfactory/wandb-jan9-ssim.csv"
csv_filename_lpips = "/home/ethanweber/nerfactory/wandb-jan9-lpips.csv"

results_psrn = csv_filename_to_results(csv_filename_psrn)
results_ssim = csv_filename_to_results(csv_filename_ssim, num_decimals=3)
results_lpips = csv_filename_to_results(csv_filename_lpips, num_decimals=3)

In [12]:

def generate_row(method, capture_names):
    # remove spaces from method
    method_text = method.replace("-", " ")
    string = f"{method_text}"
    for idx, capture_name in enumerate(capture_names):
        psnr = results_psrn[capture_name][method]
        ssim = results_ssim[capture_name][method]
        lpips = results_lpips[capture_name][method]
        if idx == len(capture_names) - 1:
            string += " & \\multicolumn{1}{r|}{" + f"{psnr} / {ssim} / {lpips}" + "}"
        else:
            string += " & \\multicolumn{1}{r}{" + f"{psnr} / {ssim} / {lpips}" + "}"
    return string

In [13]:

capture_names_row1 = [
    "Egypt",
    "person",
    "kitchen",
    "plane",
    "dozer"
]
capture_names_row2 = [
    "floating-tree",
    "aspen",
    "stump",
    "sculpture",
    "Giannini-Hall"
]

print(capture_names_row1)
for method in methods[:-1]:
    # print(generate_row(method, capture_names_row1) + ' \\\ \cline{1-1}')
    print(generate_row(method, capture_names_row1) + ' \\\ ')
print(generate_row(methods[-1], capture_names_row1) + ' \\\ \hline')

print()

print(capture_names_row2)
for method in methods[:-1]:
    # print(generate_row(method, capture_names_row2) + ' \\\ \cline{1-1}')
    print(generate_row(method, capture_names_row2) + ' \\\ ')
print(generate_row(methods[-1], capture_names_row2) + ' \\\ \hline')

['Egypt', 'person', 'kitchen', 'plane', 'dozer']
no contraction & \multicolumn{1}{r}{18.06 / 0.520 / 0.151} & \multicolumn{1}{r}{21.96 / 0.511 / 0.125} & \multicolumn{1}{r}{19.35 / 0.764 / 0.078} & \multicolumn{1}{r}{18.45 / 0.519 / 0.161} & \multicolumn{1}{r|}{19.53 / 0.786 / 0.081} \\ 
synthetic on real & \multicolumn{1}{r}{18.51 / 0.546 / 0.119} & \multicolumn{1}{r}{24.68 / 0.667 / 0.099} & \multicolumn{1}{r}{20.98 / 0.745 / 0.074} & \multicolumn{1}{r}{19.32 / 0.578 / 0.102} & \multicolumn{1}{r|}{21.41 / 0.758 / 0.080} \\ \hline

['floating-tree', 'aspen', 'stump', 'sculpture', 'Giannini-Hall']
no contraction & \multicolumn{1}{r}{18.98 / 0.679 / 0.088} & \multicolumn{1}{r}{15.80 / 0.330 / 0.342} & \multicolumn{1}{r}{20.69 / 0.704 / 0.075} & \multicolumn{1}{r}{21.43 / 0.720 / 0.073} & \multicolumn{1}{r|}{17.75 / 0.596 / 0.134} \\ 
synthetic on real & \multicolumn{1}{r}{21.13 / 0.726 / 0.072} & \multicolumn{1}{r}{15.53 / 0.301 / 0.332} & \multicolumn{1}{r}{23.86 / 0.786 / 0.053} & \multicolumn{1}{r}{22.62 / 0.706 / 0.076} & \multicolumn{1}{r|}{18.41 / 0.576 / 0.133} \\ \hline

In [ ]: