nerfstudio/scripts/benchmarking/create_tables.ipynb

In [3]:

%load_ext autoreload
%autoreload 2

In [12]:

import pandas as pd
import copy
import pprint
from benchmark_nerfstudio_paper import ablations_table_rows
from collections import defaultdict

# methods sorted in the same order we want in the paper tables
methods = [row[0] for row in ablations_table_rows]

In [34]:

def csv_filename_to_results(csv_filename, num_decimals=2, num_rows_to_consider=None):
    # code to read the csv file
    # and return a dictionary

    df = pd.read_csv(csv_filename)

    column_titles = list(df.columns.values)

    # iterate as long as the value is not None
    if num_rows_to_consider is None:
        rows = list(df.values)
    else:
        rows = list(df.values)[:num_rows_to_consider]
    valid_row = list(df.values[-1])
    for row in rows:
        for i in range(len(row)):
            value = row[i]
            if not pd.isnull(value):
                valid_row[i] = value
    last_row = valid_row
    # last_row will be the row that's valid

    from collections import defaultdict
    results = defaultdict(dict)
    for column_title, last_row_value in zip(column_titles, last_row):
        if column_title == "Step" or "__MAX" in column_title or "__MIN" in column_title:
            continue
        name = column_title.split(" - ")[0]
        capture_name, method = name.split("_")
        metric = float(last_row_value)
        # format to have 2 decimal places
        if num_decimals == 2:
            metric = f"{metric:.2f}"
        elif num_decimals == 3:
            metric = f"{metric:.3f}"
        else:
            metric = f"{metric}"
        results[capture_name][method] = metric

    # import pprint
    # pprint.pprint(results)
    # print(len(results))

    results = dict(results)
    return results

In [44]:

# the csv filenames
# these are generated from exporting CSVs from wandb graphs
csv_filename_psrn = "/home/ethanweber/nerfactory/wandb-psnr.csv"
csv_filename_ssim = "/home/ethanweber/nerfactory/wandb-ssim.csv"
csv_filename_lpips = "/home/ethanweber/nerfactory/wandb-lpips.csv"
results_psrn = csv_filename_to_results(csv_filename_psrn, num_decimals=None)
results_ssim = csv_filename_to_results(csv_filename_ssim, num_decimals=None)
results_lpips = csv_filename_to_results(csv_filename_lpips, num_decimals=None)


csv_filename_psrn = "/home/ethanweber/nerfactory/wandb-jan9-psnr.csv"
csv_filename_ssim = "/home/ethanweber/nerfactory/wandb-jan9-ssim.csv"
csv_filename_lpips = "/home/ethanweber/nerfactory/wandb-jan9-lpips.csv"
results_psrn_2 = csv_filename_to_results(csv_filename_psrn, num_decimals=None)
results_ssim_2 = csv_filename_to_results(csv_filename_ssim, num_decimals=None)
results_lpips_2 = csv_filename_to_results(csv_filename_lpips, num_decimals=None)

# merge the results
for capture_name, results in results_psrn_2.items():
    results_psrn[capture_name].update(results)
for capture_name, results in results_ssim_2.items():
    results_ssim[capture_name].update(results)
for capture_name, results in results_lpips_2.items():
    results_lpips[capture_name].update(results)

In [45]:

def get_averages_from_results(results):
    averages = defaultdict(list)
    for capture_name, methods in results.items():
        for method, metric in methods.items():
            averages[method].append(float(metric))
    for method, metrics in averages.items():
        val = sum(metrics) / len(metrics)
        averages[method] = float(f"{val:.4f}")
    return averages

In [46]:

pprint.pprint(get_averages_from_results(results_lpips))

defaultdict(<class 'list'>,
            {'1-prop-network': 0.0668,
             'l2-contraction': 0.0598,
             'nerfacto': 0.0601,
             'no-contraction': 0.1308,
             'random-background-color': 0.0599,
             'shared-prop-network': 0.0605,
             'synthetic-on-real': 0.114,
             'w/o-app': 0.0622,
             'w/o-pose': 0.0614,
             'w/o-pose-app': 0.0636})

In [47]:

def generate_row(method, capture_names):
    # remove spaces from method
    method_text = method.replace("-", " ")
    string = f"{method_text}"
    for idx, capture_name in enumerate(capture_names):
        psnr = results_psrn[capture_name][method]
        ssim = results_ssim[capture_name][method]
        lpips = results_lpips[capture_name][method]
        if idx == len(capture_names) - 1:
            string += " & \\multicolumn{1}{r|}{" + f"{psnr} / {ssim} / {lpips}" + "}"
        else:
            string += " & \\multicolumn{1}{r}{" + f"{psnr} / {ssim} / {lpips}" + "}"
    return string

In [48]:

capture_names_row1 = [
    "Egypt",
    "person",
    "kitchen",
    "plane",
    "dozer"
]
capture_names_row2 = [
    "floating-tree",
    "aspen",
    "stump",
    "sculpture",
    "Giannini-Hall"
]

print(capture_names_row1)
for method in methods[:-1]:
    # print(generate_row(method, capture_names_row1) + ' \\\ \cline{1-1}')
    print(generate_row(method, capture_names_row1) + ' \\\ ')
print(generate_row(methods[-1], capture_names_row1) + ' \\\ \hline')

print()

print(capture_names_row2)
for method in methods[:-1]:
    # print(generate_row(method, capture_names_row2) + ' \\\ \cline{1-1}')
    print(generate_row(method, capture_names_row2) + ' \\\ ')
print(generate_row(methods[-1], capture_names_row2) + ' \\\ \hline')

['Egypt', 'person', 'kitchen', 'plane', 'dozer']
nerfacto & \multicolumn{1}{r}{22.714982986450195 / 0.8457087874412537 / 0.0387668460607528} & \multicolumn{1}{r}{26.58124351501465 / 0.8588576912879944 / 0.0272847861051559} & \multicolumn{1}{r}{21.02389335632324 / 0.8730602860450745 / 0.0538232773542404} & \multicolumn{1}{r}{23.30121612548828 / 0.8020055890083313 / 0.0444588512182235} & \multicolumn{1}{r|}{21.23827743530273 / 0.8753665089607239 / 0.0552346743643283} \\ 
w/o pose & \multicolumn{1}{r}{22.709941864013672 / 0.8510463237762451 / 0.0384077467024326} & \multicolumn{1}{r}{26.875864028930664 / 0.875615656375885 / 0.0267722699791193} & \multicolumn{1}{r}{20.83899688720703 / 0.8627068996429443 / 0.0561660341918468} & \multicolumn{1}{r}{23.130054473876957 / 0.8018836975097656 / 0.0470539852976799} & \multicolumn{1}{r|}{21.35797691345215 / 0.8832465410232544 / 0.0533054806292057} \\ 
w/o app & \multicolumn{1}{r}{23.50516891479492 / 0.8450330495834351 / 0.0371274687349796} & \multicolumn{1}{r}{29.19672203063965 / 0.8823416829109192 / 0.0238535590469837} & \multicolumn{1}{r}{23.997222900390625 / 0.8697791695594788 / 0.0548945926129817} & \multicolumn{1}{r}{23.31549835205078 / 0.7850948572158813 / 0.0470350570976734} & \multicolumn{1}{r|}{24.386877059936523 / 0.8825743198394775 / 0.0454414412379264} \\ 
w/o pose app & \multicolumn{1}{r}{23.166139602661133 / 0.837310791015625 / 0.0366224125027656} & \multicolumn{1}{r}{29.16166877746582 / 0.8803766369819641 / 0.023823982104659} & \multicolumn{1}{r}{23.605825424194336 / 0.8618716597557068 / 0.0610234178602695} & \multicolumn{1}{r}{22.98196029663086 / 0.7747009992599487 / 0.0486842803657054} & \multicolumn{1}{r|}{24.1285343170166 / 0.8762974143028259 / 0.0484496317803859} \\ 
1 prop network & \multicolumn{1}{r}{22.29441261291504 / 0.8302732706069946 / 0.0457336418330669} & \multicolumn{1}{r}{26.79549789428711 / 0.8694913983345032 / 0.0297231022268533} & \multicolumn{1}{r}{20.88402557373047 / 0.8669102787971497 / 0.0568834841251373} & \multicolumn{1}{r}{23.38462257385254 / 0.8012978434562683 / 0.0477019511163234} & \multicolumn{1}{r|}{21.26655387878418 / 0.8781195878982544 / 0.0575154237449169} \\ 
l2 contraction & \multicolumn{1}{r}{22.735132217407227 / 0.8507159352302551 / 0.0394952110946178} & \multicolumn{1}{r}{26.3044490814209 / 0.8442776203155518 / 0.0297974403947591} & \multicolumn{1}{r}{20.831445693969727 / 0.870667040348053 / 0.0538116246461868} & \multicolumn{1}{r}{23.17695617675781 / 0.8045668601989746 / 0.0425091050565242} & \multicolumn{1}{r|}{21.17496681213379 / 0.8793404698371887 / 0.0562350191175937} \\ 
shared prop network & \multicolumn{1}{r}{22.72172737121582 / 0.8462676405906677 / 0.036957062780857} & \multicolumn{1}{r}{26.2338924407959 / 0.8363291621208191 / 0.030552914366126} & \multicolumn{1}{r}{20.846466064453125 / 0.8672903180122375 / 0.0565056912600994} & \multicolumn{1}{r}{23.31382369995117 / 0.8025220036506653 / 0.0451882556080818} & \multicolumn{1}{r|}{21.209640502929688 / 0.8794581890106201 / 0.0534685812890529} \\ 
random background color & \multicolumn{1}{r}{22.50684356689453 / 0.8338311910629272 / 0.0386323854327201} & \multicolumn{1}{r}{26.413747787475582 / 0.8475838899612427 / 0.0302980989217758} & \multicolumn{1}{r}{20.803401947021484 / 0.8670746684074402 / 0.0552690662443637} & \multicolumn{1}{r}{23.181974411010746 / 0.7954545021057129 / 0.0450066737830638} & \multicolumn{1}{r|}{21.30900382995605 / 0.8826950192451477 / 0.0531600639224052} \\ 
no contraction & \multicolumn{1}{r}{18.0621395111084 / 0.5198838710784912 / 0.1507363319396972} & \multicolumn{1}{r}{21.95572662353516 / 0.511165201663971 / 0.1250636577606201} & \multicolumn{1}{r}{19.351024627685547 / 0.764460563659668 / 0.0784180983901023} & \multicolumn{1}{r}{18.450420379638672 / 0.5193044543266296 / 0.1613750755786895} & \multicolumn{1}{r|}{19.532262802124023 / 0.7856844067573547 / 0.0809869393706321} \\ 
synthetic on real & \multicolumn{1}{r}{18.51077651977539 / 0.5463988184928894 / 0.1192879229784011} & \multicolumn{1}{r}{24.6822566986084 / 0.6670649647712708 / 0.0987291559576988} & \multicolumn{1}{r}{20.980388641357425 / 0.745147705078125 / 0.0741989538073539} & \multicolumn{1}{r}{19.319589614868164 / 0.5777738690376282 / 0.1021366342902183} & \multicolumn{1}{r|}{21.40807342529297 / 0.7580105066299438 / 0.0801835656166076} \\ \hline

['floating-tree', 'aspen', 'stump', 'sculpture', 'Giannini-Hall']
nerfacto & \multicolumn{1}{r}{20.39972686767578 / 0.854109525680542 / 0.0524107255041599} & \multicolumn{1}{r}{18.57237243652344 / 0.6354746222496033 / 0.1751366555690765} & \multicolumn{1}{r}{21.928449630737305 / 0.7976410388946533 / 0.0387384481728076} & \multicolumn{1}{r}{22.58584213256836 / 0.7889150381088257 / 0.0407779291272163} & \multicolumn{1}{r|}{19.99639129638672 / 0.7731102108955383 / 0.0745075196027755} \\ 
w/o pose & \multicolumn{1}{r}{20.298728942871094 / 0.8402881622314453 / 0.0546704977750778} & \multicolumn{1}{r}{18.61675453186035 / 0.6382865905761719 / 0.1753666847944259} & \multicolumn{1}{r}{21.633838653564453 / 0.7653179168701172 / 0.0428420193493366} & \multicolumn{1}{r}{22.506526947021484 / 0.7845871448516846 / 0.0455470941960811} & \multicolumn{1}{r|}{19.97289276123047 / 0.7722745537757874 / 0.0739260166883468} \\ 
w/o app & \multicolumn{1}{r}{22.46498680114746 / 0.8535845279693604 / 0.0616657771170139} & \multicolumn{1}{r}{18.27329635620117 / 0.6129483580589294 / 0.1905485093593597} & \multicolumn{1}{r}{25.68264579772949 / 0.8736064434051514 / 0.0317401029169559} & \multicolumn{1}{r}{23.268043518066406 / 0.7738019227981567 / 0.0487339384853839} & \multicolumn{1}{r|}{21.44996833801269 / 0.7652416825294495 / 0.0814294144511222} \\ 
w/o pose app & \multicolumn{1}{r}{22.154300689697266 / 0.8340064287185669 / 0.0661337822675705} & \multicolumn{1}{r}{18.28131675720215 / 0.611172080039978 / 0.1861454993486404} & \multicolumn{1}{r}{25.562631607055664 / 0.8677808046340942 / 0.0340047031641006} & \multicolumn{1}{r}{22.98941802978516 / 0.76247638463974 / 0.0532486699521541} & \multicolumn{1}{r|}{21.344554901123047 / 0.7619099020957947 / 0.0781947895884513} \\ 
1 prop network & \multicolumn{1}{r}{20.28233528137207 / 0.8498139977455139 / 0.0570690296590328} & \multicolumn{1}{r}{18.41587829589844 / 0.6070924401283264 / 0.2094678878784179} & \multicolumn{1}{r}{22.85092544555664 / 0.8734478950500488 / 0.0329417251050472} & \multicolumn{1}{r}{22.60772132873535 / 0.8046638369560242 / 0.0431603491306304} & \multicolumn{1}{r|}{19.89984703063965 / 0.759652853012085 / 0.0880114957690239} \\ 
l2 contraction & \multicolumn{1}{r}{20.49176979064941 / 0.8501678109169006 / 0.0527788065373897} & \multicolumn{1}{r}{18.66337013244629 / 0.6445528268814087 / 0.1680227816104889} & \multicolumn{1}{r}{22.26547622680664 / 0.8150904178619385 / 0.0378103740513324} & \multicolumn{1}{r}{22.462072372436523 / 0.7815755009651184 / 0.0426461324095726} & \multicolumn{1}{r|}{19.985918045043945 / 0.7727018594741821 / 0.0751706212759018} \\ 
shared prop network & \multicolumn{1}{r}{20.333086013793945 / 0.8504053950309753 / 0.0537901557981967} & \multicolumn{1}{r}{18.6508731842041 / 0.6390713453292847 / 0.1719826906919479} & \multicolumn{1}{r}{22.207321166992188 / 0.805411696434021 / 0.0403469167649745} & \multicolumn{1}{r}{22.50098991394043 / 0.7922176718711853 / 0.0416620261967182} & \multicolumn{1}{r|}{19.98356437683105 / 0.7727880477905273 / 0.0745216161012649} \\ 
random background color & \multicolumn{1}{r}{20.243900299072266 / 0.8465315699577332 / 0.0529390051960945} & \multicolumn{1}{r}{18.56953430175781 / 0.636640191078186 / 0.1718863695859909} & \multicolumn{1}{r}{22.578887939453125 / 0.8352341055870056 / 0.0351601541042327} & \multicolumn{1}{r}{22.478111267089844 / 0.7781183123588562 / 0.0417613536119461} & \multicolumn{1}{r|}{20.105976104736328 / 0.7749199867248535 / 0.0752039030194282} \\ 
no contraction & \multicolumn{1}{r}{18.98202896118164 / 0.6787521839141846 / 0.0879055261611938} & \multicolumn{1}{r}{15.803391456604004 / 0.3301353752613067 / 0.3417288959026336} & \multicolumn{1}{r}{20.686363220214844 / 0.7041175365447998 / 0.0748037695884704} & \multicolumn{1}{r}{21.432249069213867 / 0.7197808623313904 / 0.0730623304843902} & \multicolumn{1}{r|}{17.751110076904297 / 0.5955697298049927 / 0.1335323601961136} \\ 
synthetic on real & \multicolumn{1}{r}{21.130878448486328 / 0.7259055376052856 / 0.0717189684510231} & \multicolumn{1}{r}{15.527592658996582 / 0.3011459112167358 / 0.3317984342575073} & \multicolumn{1}{r}{23.858999252319336 / 0.7855907678604126 / 0.0529582537710666} & \multicolumn{1}{r}{22.620250701904297 / 0.7056115865707397 / 0.0756969302892685} & \multicolumn{1}{r|}{18.407384872436523 / 0.5756094455718994 / 0.1331061422824859} \\ \hline

In [13]:

# mipnerf360 experiments

csv_filename_psrn = "/home/ethanweber/nerfactory/mipnerf360-nerfacto-psnr.csv"
csv_filename_ssim = "/home/ethanweber/nerfactory/mipnerf360-nerfacto-ssim.csv"
csv_filename_lpips = "/home/ethanweber/nerfactory/mipnerf360-nerfacto-lpips.csv"

csv_filename_psrn = "/home/ethanweber/nerfactory/mipnerf360-instant-ngp-psnr.csv"
csv_filename_ssim = "/home/ethanweber/nerfactory/mipnerf360-instant-ngp-ssim.csv"
csv_filename_lpips = "/home/ethanweber/nerfactory/mipnerf360-instant-ngp-lpips.csv"

results_psrn = csv_filename_to_results(csv_filename_psrn, num_rows_to_consider=1)
results_ssim = csv_filename_to_results(csv_filename_ssim, num_decimals=3, num_rows_to_consider=1)
results_lpips = csv_filename_to_results(csv_filename_lpips, num_decimals=3, num_rows_to_consider=1)

In [11]:

mipnerf360_nerfacto_capture_names = ["bicycle", "garden", "stump", "room", "counter", "kitchen", "bonsai"]

print("psnr")
s = [(key, results_psrn[key]) for key in mipnerf360_nerfacto_capture_names]
pprint.pprint(s)
print()

print("ssim")
s = [(key, results_ssim[key]) for key in mipnerf360_nerfacto_capture_names]
pprint.pprint(s)
print()

print("lpips")
s = [(key, results_lpips[key]) for key in mipnerf360_nerfacto_capture_names]
pprint.pprint(s)
print()

psnr
[('bicycle', {'instant-ngp-w/o-pose-app': '19.92'}),
 ('garden', {'instant-ngp-w/o-pose-app': '22.68'}),
 ('stump', {'instant-ngp-w/o-pose-app': '20.25'}),
 ('room', {'instant-ngp-w/o-pose-app': '28.04'}),
 ('counter', {'instant-ngp-w/o-pose-app': '23.59'}),
 ('kitchen', {'instant-ngp-w/o-pose-app': '24.61'}),
 ('bonsai', {'instant-ngp-w/o-pose-app': '25.67'})]

ssim
[('bicycle', {'instant-ngp-w/o-pose-app': '0.448'}),
 ('garden', {'instant-ngp-w/o-pose-app': '0.542'}),
 ('stump', {'instant-ngp-w/o-pose-app': '0.465'}),
 ('room', {'instant-ngp-w/o-pose-app': '0.828'}),
 ('counter', {'instant-ngp-w/o-pose-app': '0.701'}),
 ('kitchen', {'instant-ngp-w/o-pose-app': '0.642'}),
 ('bonsai', {'instant-ngp-w/o-pose-app': '0.792'})]

lpips
[('bicycle', {'instant-ngp-w/o-pose-app': '0.449'}),
 ('garden', {'instant-ngp-w/o-pose-app': '0.402'}),
 ('stump', {'instant-ngp-w/o-pose-app': '0.440'}),
 ('room', {'instant-ngp-w/o-pose-app': '0.170'}),
 ('counter', {'instant-ngp-w/o-pose-app': '0.271'}),
 ('kitchen', {'instant-ngp-w/o-pose-app': '0.281'}),
 ('bonsai', {'instant-ngp-w/o-pose-app': '0.186'})]

In [14]:

mipnerf360_nerfacto_capture_names = ["bicycle", "garden", "stump", "room", "counter", "kitchen", "bonsai"]

print("psnr")
s = [(key, results_psrn[key]) for key in mipnerf360_nerfacto_capture_names]
pprint.pprint(s)
print()

print("ssim")
s = [(key, results_ssim[key]) for key in mipnerf360_nerfacto_capture_names]
pprint.pprint(s)
print()

print("lpips")
s = [(key, results_lpips[key]) for key in mipnerf360_nerfacto_capture_names]
pprint.pprint(s)
print()

psnr
[('bicycle', {'instant-ngp-w/o-pose-app': '18.78'}),
 ('garden', {'instant-ngp-w/o-pose-app': '20.82'}),
 ('stump', {'instant-ngp-w/o-pose-app': '19.73'}),
 ('room', {'instant-ngp-w/o-pose-app': '25.26'}),
 ('counter', {'instant-ngp-w/o-pose-app': '22.09'}),
 ('kitchen', {'instant-ngp-w/o-pose-app': '22.82'}),
 ('bonsai', {'instant-ngp-w/o-pose-app': '23.86'})]

ssim
[('bicycle', {'instant-ngp-w/o-pose-app': '0.389'}),
 ('garden', {'instant-ngp-w/o-pose-app': '0.390'}),
 ('stump', {'instant-ngp-w/o-pose-app': '0.408'}),
 ('room', {'instant-ngp-w/o-pose-app': '0.749'}),
 ('counter', {'instant-ngp-w/o-pose-app': '0.621'}),
 ('kitchen', {'instant-ngp-w/o-pose-app': '0.530'}),
 ('bonsai', {'instant-ngp-w/o-pose-app': '0.665'})]

lpips
[('bicycle', {'instant-ngp-w/o-pose-app': '0.620'}),
 ('garden', {'instant-ngp-w/o-pose-app': '0.619'}),
 ('stump', {'instant-ngp-w/o-pose-app': '0.607'}),
 ('room', {'instant-ngp-w/o-pose-app': '0.311'}),
 ('counter', {'instant-ngp-w/o-pose-app': '0.417'}),
 ('kitchen', {'instant-ngp-w/o-pose-app': '0.427'}),
 ('bonsai', {'instant-ngp-w/o-pose-app': '0.382'})]

In [ ]: