working on abllations

scripts/benchmarking/benchmark_nerfacto.py

@@ -4,6 +4,11 @@ Process the Mipnerf360 dataset.
 
 from nerfstudio.utils.scripts import run_command
 
+import GPUtil
+import time
+import threading
+
+
 capture_names = [
     "Egypt",
     "person",
@@ -17,36 +22,87 @@ capture_names = [
     "Giannini-Hall",
 ]
 table_rows = [
-    ("nerfacto", ""),
-    ("w/o-pose", "--pipeline.datamanager.camera-optimizer.mode off"),
-    ("w/o-app", "--pipeline.model.use-appearance-embedding False"),
+    ("nerfacto", "--pipeline.eval_optimize_cameras True --pipeline.eval_optimize_appearance True"),
+    (
+        "w/o-pose",
+        "--pipeline.eval_optimize_cameras True --pipeline.eval_optimize_appearance True --pipeline.datamanager.camera-optimizer.mode off",
+    ),
+    (
+        "w/o-app",
+        "--pipeline.eval_optimize_cameras True --pipeline.eval_optimize_appearance False --pipeline.model.use-appearance-embedding False",
+    ),
     (
         "w/o-pose-app",
-        "--pipeline.datamanager.camera-optimizer.mode off --pipeline.model.use-appearance-embedding False",
+        "--pipeline.eval_optimize_cameras True --pipeline.eval_optimize_appearance False --pipeline.datamanager.camera-optimizer.mode off --pipeline.model.use-appearance-embedding False",
     ),
     (
         "1-prop-network",
-        "--pipeline.model.num-proposal-samples-per-ray \"256\" --pipeline.model.num_proposal_iterations 1",
+        '--pipeline.eval_optimize_cameras True --pipeline.eval_optimize_appearance True --pipeline.model.num-proposal-samples-per-ray "256" --pipeline.model.num_proposal_iterations 1',
+    ),
+    (
+        "l2-contraction",
+        "--pipeline.eval_optimize_cameras True --pipeline.eval_optimize_appearance True --pipeline.model.scene-contraction-norm l2",
+    ),
+    (
+        "shared-prop-network",
+        "--pipeline.eval_optimize_cameras True --pipeline.eval_optimize_appearance True --pipeline.model.use-same-proposal-network True",
+    ),
+    (
+        "random-background-color",
+        "--pipeline.eval_optimize_cameras True --pipeline.eval_optimize_appearance True --pipeline.model.background-color random",
     ),
 ]
 
 # 30K iterations
 
+# make a list of all the jobs that need to be fun
+jobs = []
 for capture_name in capture_names:
     for table_row_name, table_row_command in table_rows:
-
-        # TODO: find an available GPU
-        gpu = 0
-
         command = " ".join(
             (
-                # f"CUDA_VISIBLE_DEVICES={gpu}",
                 f"ns-train nerfacto",
                 "--vis wandb",
                 f"--data data/nerfstudio/{capture_name}",
-                "--pipeline.eval_optimize_cameras True --pipeline.eval_optimize_appearance True",
-                f"--wandb-name {capture_name}-{table_row_name} --experiment-name {capture_name}-{table_row_name}",
+                "--output-dir outputs/nerfacto-ablations",
+                "--trainer.steps-per-eval-batch 0 --trainer.steps-per-eval-image 0",
+                "--trainer.steps-per-eval-all-images 5000 --trainer.max-num-iterations 30001",
+                f"--wandb-name {capture_name}_{table_row_name}",
+                f"--experiment-name {capture_name}_{table_row_name}",
                 table_row_command,
             )
         )
-        print(command)
+        jobs.append(command)
+
+while jobs:
+
+    # check which GPUs have capacity to run these jobs
+    """Returns the available GPUs."""
+    gpu_devices_available = GPUtil.getAvailable(order="first", limit=10, maxMemory=0.1)
+
+    print("Available GPUs: ", gpu_devices_available)
+
+    # thread list
+    threads = []
+    while gpu_devices_available and jobs:
+        gpu = gpu_devices_available.pop(0)
+        command = f"CUDA_VISIBLE_DEVICES={gpu} " + jobs.pop(0)
+
+        def task():
+            print("Starting command: ", command)
+            out = run_command(command, verbose=False)
+            # time.sleep(5)
+            print("Finished command: ", command)
+
+        threads.append(threading.Thread(target=task))
+        threads[-1].start()
+
+        # NOTE(ethan): need a delay otherwise the wandb/tensorboard naming is messed up
+        # not sure why?
+        time.sleep(5)
+
+    # wait for all threads to finish
+    for t in threads:
+        t.join()
+
+    print("Finished all threads")

scripts/benchmarking/create_tables.py

@@ -0,0 +1,36 @@
+"""
+Code to create the tables.
+"""
+
+import pandas as pd
+
+# from nerfstudio.scripts.benchmarking.benchmark_nerfacto import table_rows
+
+# the csv filenames
+csv_filename = "wandb-psnr.csv"
+
+# code to read the csv file
+
+df = pd.read_csv(csv_filename)
+
+column_titles = list(df.columns.values)
+last_row = list(df.values[-1])
+
+print(len(column_titles))
+print(len(last_row))
+
+from collections import defaultdict
+results = defaultdict(dict)
+for column_title, last_row_value in zip(column_titles, last_row):
+    if column_title == "Step" or "__MAX" in column_title or "__MIN" in column_title:
+        continue
+    name = column_title.split(" - ")[0]
+    capture_name, method = name.split("_")
+    metric = float(last_row_value)
+    # format to have 2 decimal places
+    metric = f"{metric:.2f}"
+    results[capture_name][method] = metric
+
+import pprint
+pprint.pprint(results)
+print(len(results))

scripts/downloads/download_data.py

@@ -97,6 +97,15 @@ nerfstudio_dataset = [
     "sculpture",
     "Giannini-Hall",
 ]
+mipnerf360_dataset = [
+    "mipnerf-360-bicycle",
+    "mipnerf-360-garden",
+    "mipnerf-360-stump",
+    "mipnerf-360-room",
+    "mipnerf-360-counter",
+    "mipnerf-360-kitchen",
+    "mipnerf-360-bonsai",
+]
 nerfstudio_file_ids = {
     "bww_entrance": grab_file_id("https://drive.google.com/file/d/1ylkRHtfB3n3IRLf2wplpfxzPTq7nES9I/view?usp=sharing"),
     "campanile": grab_file_id("https://drive.google.com/file/d/13aOfGJRRH05pOOk9ikYGTwqFc2L1xskU/view?usp=sharing"),
@@ -106,6 +115,7 @@ nerfstudio_file_ids = {
     "redwoods2": grab_file_id("https://drive.google.com/file/d/1rg-4NoXT8p6vkmbWxMOY6PSG4j3rfcJ8/view?usp=sharing"),
     "storefront": grab_file_id("https://drive.google.com/file/d/16b792AguPZWDA_YC4igKCwXJqW0Tb21o/view?usp=sharing"),
     "vegetation": grab_file_id("https://drive.google.com/file/d/1wBhLQ2odycrtU39y2akVurXEAt9SsVI3/view?usp=sharing"),
+    # Nerfstudio Dataset
     "Egypt": "https://data.nerf.studio/nerfstudio/Egypt.zip",
     "person": "https://data.nerf.studio/nerfstudio/person.zip",
     "kitchen": "https://data.nerf.studio/nerfstudio/kitchen.zip",
@@ -116,8 +126,17 @@ nerfstudio_file_ids = {
     "stump": "https://data.nerf.studio/nerfstudio/stump.zip",
     "sculpture": "https://data.nerf.studio/nerfstudio/sculpture.zip",
     "Giannini-Hall": "https://data.nerf.studio/nerfstudio/Giannini-Hall.zip",
+    # MipNeRF-360 Dataset. TODO: add these links
+    "mipnerf360-bicycle": "",
+    "mipnerf360-garden": "",
+    "mipnerf360-stump": "",
+    "mipnerf360-room": "",
+    "mipnerf360-counter": "",
+    "mipnerf360-kitchen": "",
+    "mipnerf360-bonsai": "",
     "all": None,
     "nerfstudio-dataset": nerfstudio_dataset,
+    "mipnerf360-dataset": mipnerf360_dataset,
 }
 
 NerfstudioCaptureName = tyro.extras.literal_type_from_choices(nerfstudio_file_ids.keys())