nerfstudio/scripts/benchmarking/create_tables.ipynb

In [3]:

%load_ext autoreload
%autoreload 2

In [4]:

import pandas as pd
import copy
import pprint
from benchmark_nerfstudio_paper import table_rows

# methods sorted in the same order we want in the paper tables
methods = [row[0] for row in table_rows]

---------------------------------------------------------------------------
ImportError                               Traceback (most recent call last)
Cell In [4], line 4
      2 import copy
      3 import pprint
----> 4 from benchmark_nerfstudio_paper import table_rows
      6 # methods sorted in the same order we want in the paper tables
      7 methods = [row[0] for row in table_rows]

ImportError: cannot import name 'table_rows' from 'benchmark_nerfstudio_paper' (/home/ethanweber/nerfactory/scripts/benchmarking/benchmark_nerfstudio_paper.py)

In [5]:

def csv_filename_to_results(csv_filename, num_decimals=2, num_rows_to_consider=None):
    # code to read the csv file
    # and return a dictionary

    df = pd.read_csv(csv_filename)

    column_titles = list(df.columns.values)

    # iterate as long as the value is not None
    if num_rows_to_consider is None:
        rows = list(df.values)
    else:
        rows = list(df.values)[:num_rows_to_consider]
    valid_row = list(df.values[-1])
    for row in rows:
        for i in range(len(row)):
            value = row[i]
            if not pd.isnull(value):
                valid_row[i] = value
    last_row = valid_row
    # last_row will be the row that's valid

    from collections import defaultdict
    results = defaultdict(dict)
    for column_title, last_row_value in zip(column_titles, last_row):
        if column_title == "Step" or "__MAX" in column_title or "__MIN" in column_title:
            continue
        name = column_title.split(" - ")[0]
        capture_name, method = name.split("_")
        metric = float(last_row_value)
        # format to have 2 decimal places
        if num_decimals == 2:
            metric = f"{metric:.2f}"
        elif num_decimals == 3:
            metric = f"{metric:.3f}"
        results[capture_name][method] = metric

    # import pprint
    # pprint.pprint(results)
    # print(len(results))

    results = dict(results)
    return results

In [11]:

# the csv filenames
# these are generated from exporting CSVs from wandb graphs
# csv_filename_psrn = "/home/ethanweber/nerfactory/wandb-psnr.csv"
# csv_filename_ssim = "/home/ethanweber/nerfactory/wandb-ssim.csv"
# csv_filename_lpips = "/home/ethanweber/nerfactory/wandb-lpips.csv"
csv_filename_psrn = "/home/ethanweber/nerfactory/wandb-jan9-psnr.csv"
csv_filename_ssim = "/home/ethanweber/nerfactory/wandb-jan9-ssim.csv"
csv_filename_lpips = "/home/ethanweber/nerfactory/wandb-jan9-lpips.csv"

results_psrn = csv_filename_to_results(csv_filename_psrn)
results_ssim = csv_filename_to_results(csv_filename_ssim, num_decimals=3)
results_lpips = csv_filename_to_results(csv_filename_lpips, num_decimals=3)

In [12]:

def generate_row(method, capture_names):
    # remove spaces from method
    method_text = method.replace("-", " ")
    string = f"{method_text}"
    for idx, capture_name in enumerate(capture_names):
        psnr = results_psrn[capture_name][method]
        ssim = results_ssim[capture_name][method]
        lpips = results_lpips[capture_name][method]
        if idx == len(capture_names) - 1:
            string += " & \\multicolumn{1}{r|}{" + f"{psnr} / {ssim} / {lpips}" + "}"
        else:
            string += " & \\multicolumn{1}{r}{" + f"{psnr} / {ssim} / {lpips}" + "}"
    return string

In [13]:

capture_names_row1 = [
    "Egypt",
    "person",
    "kitchen",
    "plane",
    "dozer"
]
capture_names_row2 = [
    "floating-tree",
    "aspen",
    "stump",
    "sculpture",
    "Giannini-Hall"
]

print(capture_names_row1)
for method in methods[:-1]:
    # print(generate_row(method, capture_names_row1) + ' \\\ \cline{1-1}')
    print(generate_row(method, capture_names_row1) + ' \\\ ')
print(generate_row(methods[-1], capture_names_row1) + ' \\\ \hline')

print()

print(capture_names_row2)
for method in methods[:-1]:
    # print(generate_row(method, capture_names_row2) + ' \\\ \cline{1-1}')
    print(generate_row(method, capture_names_row2) + ' \\\ ')
print(generate_row(methods[-1], capture_names_row2) + ' \\\ \hline')

['Egypt', 'person', 'kitchen', 'plane', 'dozer']
no contraction & \multicolumn{1}{r}{18.06 / 0.520 / 0.151} & \multicolumn{1}{r}{21.96 / 0.511 / 0.125} & \multicolumn{1}{r}{19.35 / 0.764 / 0.078} & \multicolumn{1}{r}{18.45 / 0.519 / 0.161} & \multicolumn{1}{r|}{19.53 / 0.786 / 0.081} \\ 
synthetic on real & \multicolumn{1}{r}{18.51 / 0.546 / 0.119} & \multicolumn{1}{r}{24.68 / 0.667 / 0.099} & \multicolumn{1}{r}{20.98 / 0.745 / 0.074} & \multicolumn{1}{r}{19.32 / 0.578 / 0.102} & \multicolumn{1}{r|}{21.41 / 0.758 / 0.080} \\ \hline

['floating-tree', 'aspen', 'stump', 'sculpture', 'Giannini-Hall']
no contraction & \multicolumn{1}{r}{18.98 / 0.679 / 0.088} & \multicolumn{1}{r}{15.80 / 0.330 / 0.342} & \multicolumn{1}{r}{20.69 / 0.704 / 0.075} & \multicolumn{1}{r}{21.43 / 0.720 / 0.073} & \multicolumn{1}{r|}{17.75 / 0.596 / 0.134} \\ 
synthetic on real & \multicolumn{1}{r}{21.13 / 0.726 / 0.072} & \multicolumn{1}{r}{15.53 / 0.301 / 0.332} & \multicolumn{1}{r}{23.86 / 0.786 / 0.053} & \multicolumn{1}{r}{22.62 / 0.706 / 0.076} & \multicolumn{1}{r|}{18.41 / 0.576 / 0.133} \\ \hline

In [13]:

csv_filename_psrn = "/home/ethanweber/nerfactory/mipnerf360-instant-ngp-psnr.csv"
csv_filename_ssim = "/home/ethanweber/nerfactory/mipnerf360-instant-ngp-ssim.csv"
csv_filename_lpips = "/home/ethanweber/nerfactory/mipnerf360-instant-ngp-lpips.csv"

results_psrn = csv_filename_to_results(csv_filename_psrn, num_rows_to_consider=1)
results_ssim = csv_filename_to_results(csv_filename_ssim, num_decimals=3, num_rows_to_consider=1)
results_lpips = csv_filename_to_results(csv_filename_lpips, num_decimals=3, num_rows_to_consider=1)

In [11]:

mipnerf360_nerfacto_capture_names = ["bicycle", "garden", "stump", "room", "counter", "kitchen", "bonsai"]

print("psnr")
s = [(key, results_psrn[key]) for key in mipnerf360_nerfacto_capture_names]
pprint.pprint(s)
print()

print("ssim")
s = [(key, results_ssim[key]) for key in mipnerf360_nerfacto_capture_names]
pprint.pprint(s)
print()

print("lpips")
s = [(key, results_lpips[key]) for key in mipnerf360_nerfacto_capture_names]
pprint.pprint(s)
print()

psnr
[('bicycle', {'instant-ngp-w/o-pose-app': '19.92'}),
 ('garden', {'instant-ngp-w/o-pose-app': '22.68'}),
 ('stump', {'instant-ngp-w/o-pose-app': '20.25'}),
 ('room', {'instant-ngp-w/o-pose-app': '28.04'}),
 ('counter', {'instant-ngp-w/o-pose-app': '23.59'}),
 ('kitchen', {'instant-ngp-w/o-pose-app': '24.61'}),
 ('bonsai', {'instant-ngp-w/o-pose-app': '25.67'})]

ssim
[('bicycle', {'instant-ngp-w/o-pose-app': '0.448'}),
 ('garden', {'instant-ngp-w/o-pose-app': '0.542'}),
 ('stump', {'instant-ngp-w/o-pose-app': '0.465'}),
 ('room', {'instant-ngp-w/o-pose-app': '0.828'}),
 ('counter', {'instant-ngp-w/o-pose-app': '0.701'}),
 ('kitchen', {'instant-ngp-w/o-pose-app': '0.642'}),
 ('bonsai', {'instant-ngp-w/o-pose-app': '0.792'})]

lpips
[('bicycle', {'instant-ngp-w/o-pose-app': '0.449'}),
 ('garden', {'instant-ngp-w/o-pose-app': '0.402'}),
 ('stump', {'instant-ngp-w/o-pose-app': '0.440'}),
 ('room', {'instant-ngp-w/o-pose-app': '0.170'}),
 ('counter', {'instant-ngp-w/o-pose-app': '0.271'}),
 ('kitchen', {'instant-ngp-w/o-pose-app': '0.281'}),
 ('bonsai', {'instant-ngp-w/o-pose-app': '0.186'})]

In [14]:

mipnerf360_nerfacto_capture_names = ["bicycle", "garden", "stump", "room", "counter", "kitchen", "bonsai"]

print("psnr")
s = [(key, results_psrn[key]) for key in mipnerf360_nerfacto_capture_names]
pprint.pprint(s)
print()

print("ssim")
s = [(key, results_ssim[key]) for key in mipnerf360_nerfacto_capture_names]
pprint.pprint(s)
print()

print("lpips")
s = [(key, results_lpips[key]) for key in mipnerf360_nerfacto_capture_names]
pprint.pprint(s)
print()

psnr
[('bicycle', {'instant-ngp-w/o-pose-app': '18.78'}),
 ('garden', {'instant-ngp-w/o-pose-app': '20.82'}),
 ('stump', {'instant-ngp-w/o-pose-app': '19.73'}),
 ('room', {'instant-ngp-w/o-pose-app': '25.26'}),
 ('counter', {'instant-ngp-w/o-pose-app': '22.09'}),
 ('kitchen', {'instant-ngp-w/o-pose-app': '22.82'}),
 ('bonsai', {'instant-ngp-w/o-pose-app': '23.86'})]

ssim
[('bicycle', {'instant-ngp-w/o-pose-app': '0.389'}),
 ('garden', {'instant-ngp-w/o-pose-app': '0.390'}),
 ('stump', {'instant-ngp-w/o-pose-app': '0.408'}),
 ('room', {'instant-ngp-w/o-pose-app': '0.749'}),
 ('counter', {'instant-ngp-w/o-pose-app': '0.621'}),
 ('kitchen', {'instant-ngp-w/o-pose-app': '0.530'}),
 ('bonsai', {'instant-ngp-w/o-pose-app': '0.665'})]

lpips
[('bicycle', {'instant-ngp-w/o-pose-app': '0.620'}),
 ('garden', {'instant-ngp-w/o-pose-app': '0.619'}),
 ('stump', {'instant-ngp-w/o-pose-app': '0.607'}),
 ('room', {'instant-ngp-w/o-pose-app': '0.311'}),
 ('counter', {'instant-ngp-w/o-pose-app': '0.417'}),
 ('kitchen', {'instant-ngp-w/o-pose-app': '0.427'}),
 ('bonsai', {'instant-ngp-w/o-pose-app': '0.382'})]

In [ ]: