Config restructure (#283)

* starting docs for pipelines

* working on an example

* clearing output

* starting to implement config changes and add documentation

* wip: removed cicular imports; porting basic config into python; todo: get python setups workin

* circular imports again

* wip: pausing at dataloader sort of working

* break for today

* running vanilla

* wip need to get all the config docs working

* laptop about to die

* instant ngp working

* added all configs for all methods; not tested yet: [mipnerf360, nerfw, semantic_nerf]

* unpacking later

* done with pylinting

* mostly done. just need to do docs and one test

* added import order to docs

* fixing near/far plane and other config things

* making loggers also instantiable via configs

* docs

* lint

* making --help work with dcargs

* moving configs back into flat

* convert list to tuples; fixed post init function with mutable params

* lint

* made run_eval and benchmarking scripts compatible with new config/dcargs

* lint

* added option to load configs from presaved config

Co-authored-by: evonneng <evonne_ng@berkeley.edu>
This commit is contained in:
Ethan Weberandevonneng authored and GitHub committed 2022-08-26 17:25:10 -07:00
1 parent e0856e3afc
commit 7e9a56fac4
46 files changed
+987 -916

No files matched your search

+3 -5
View File
@@ -59,11 +59,9 @@ for dataset in ${DATASETS[@]}; do
export CUDA_VISIBLE_DEVICES=${GPU_IDX[$idx]}
base_config_name=${config_name/"graph_"/""}
checkpoint_dir="${output_dir}/blender_${dataset}_${month}-${date}-${year}/${base_config_name}/${year}-${month}-${date}_${seconds}/nerfactory_models/"
python scripts/run_eval.py --config-name ${config_name} \
--checkpoint-dir ${checkpoint_dir} \
--rendered-output-name rgb \
--output-filename ${output_dir}/${base_config_name}/blender_${dataset}_${month}-${date}-${year}_${seconds}.json \
--config-overrides pipeline.dataloader.train_dataset.data_directory=data/blender/${dataset} &
python scripts/run_eval.py ${config_name} \
--eval.checkpoint-dir=${checkpoint_dir} \
--eval.output-filename=${output_dir}/${base_config_name}/blender_${dataset}_${month}-${date}-${year}_${seconds}.json &
echo "Launched ${checkpoint_dir} on gpu ${GPU_IDX[$idx]}"
# update gpu
+9 -10
View File
@@ -3,7 +3,7 @@
helpFunction_launch_train()
{
echo "Usage: $0 -c config_name -g gpu_list"
echo -e "\t-c name of config to benchmark (e.g. graph_mipnerf, graph_instant_ngp)"
echo -e "\t-c name of config to benchmark (e.g. mipnerf, instant_ngp)"
echo -e "\t-g list of space-separated gpu numbers to launch train on (e.g. 0 2 4 5)"
exit 1 # Exit program after printing help
}
@@ -53,15 +53,14 @@ len=${#GPU_IDX[@]}
for dataset in ${DATASETS[@]}; do
export CUDA_VISIBLE_DEVICES=${GPU_IDX[$idx]}
python scripts/run_train.py \
--config-name ${config_name} \
'~logging.writer.LocalWriter' \
pipeline.dataloader.train_dataset.data_directory=data/blender/${dataset} \
experiment_name=blender_${dataset}_${tag} \
trainer.model_dir=nerfactory_models/ \
trainer.steps_per_save=25000 \
trainer.max_num_iterations=2000000 \
viewer.enable=False \
logging.enable_profiler=False &
${config_name} \
--logging.writer.2.no-enable \
--pipeline.dataloader.train-dataset.data-directory=data/blender/${dataset} \
--experiment-name=blender_${dataset}_${tag} \
--trainer.model-dir=nerfactory_models/ \
--trainer.steps-per-save=16000 \
--trainer.max-num-iterations=16500 \
--logging.no-enable-profiler &
echo "Launched ${config_name} ${dataset} on gpu ${GPU_IDX[$idx]}, ${tag}"
# update gpu
+52 -61
View File
@@ -1,16 +1,15 @@
"""
run_eval.py
"""
import enum
import json
import logging
import os
from typing import Dict, List, Optional
import pickle
from pathlib import Path
from typing import Dict
import dcargs
import mediapy as media
import torch
from hydra import compose, initialize
from omegaconf import DictConfig
from tqdm import tqdm
from nerfactory.cameras.camera_paths import (
@@ -18,12 +17,14 @@ from nerfactory.cameras.camera_paths import (
get_interpolated_camera_path,
get_spiral_path,
)
from nerfactory.pipelines.base import Pipeline, setup_pipeline
from nerfactory.utils import io as io_utils
from nerfactory.utils.config import setup_config
from nerfactory.configs import base as cfg
from nerfactory.configs.utils import cli_from_base_configs
from nerfactory.pipelines.base import Pipeline
from nerfactory.utils.misc import human_format
from nerfactory.utils.writer import TimeWriter
logging.basicConfig(format="[%(filename)s:%(lineno)d] %(message)s", level=logging.INFO)
def _update_avg(prev_avg: float, new_val: float, step: int) -> float:
"""helper to calculate the running average
@@ -39,7 +40,7 @@ def _update_avg(prev_avg: float, new_val: float, step: int) -> float:
return (step * prev_avg + new_val) / (step + 1)
def _load_checkpoint(config: DictConfig, pipeline: Pipeline) -> str:
def _load_checkpoint(config: cfg.TrainerConfig, pipeline: Pipeline) -> Path:
"""Helper function to load checkpointed pipeline
Args:
@@ -53,8 +54,8 @@ def _load_checkpoint(config: DictConfig, pipeline: Pipeline) -> str:
load_step = sorted(int(x[x.find("-") + 1 : x.find(".")]) for x in os.listdir(config.load_dir))[-1]
else:
load_step = config.load_step
load_path = os.path.join(config.load_dir, f"step-{load_step:09d}.ckpt")
assert os.path.exists(load_path), f"Checkpoint {load_path} does not exist"
load_path = config.load_dir / f"step-{load_step:09d}.ckpt"
assert load_path.exists(), f"Checkpoint {load_path} does not exist"
loaded_state = torch.load(load_path, map_location="cpu")
pipeline.load_pipeline(loaded_state["pipeline"])
print(f"done loading checkpoint from {load_path}")
@@ -88,7 +89,7 @@ def render_stats_dict(pipeline: Pipeline) -> Dict[str, float]:
def render_trajectory_video(
pipeline: Pipeline,
camera_path: CameraPath,
output_filename: str,
output_filename: Path,
rendered_output_name: str,
rendered_resolution_scaling_factor: float = 1.0,
num_rays_per_chunk: int = 4096,
@@ -120,30 +121,7 @@ def render_trajectory_video(
media.write_video(output_filename, images, fps=fps)
class MethodType(enum.Enum):
"""Enum for the method type."""
PSNR = enum.auto()
TRAJ = enum.auto()
class TrajectoryType(enum.Enum):
"""Enum for the trajectory type."""
SPIRAL = enum.auto()
INTERP = enum.auto()
def main(
config_name: str,
checkpoint_dir: str,
rendered_output_name: str,
method: MethodType = MethodType.PSNR,
traj: TrajectoryType = TrajectoryType.SPIRAL,
output_filename: str = "output.json",
rendered_resolution_scaling_factor: float = 1.0,
config_overrides: Optional[List[str]] = None,
):
def main(config: cfg.Config):
"""Evaluate trained model. This evaluation can either render a trajectory or compute the eval psnr.
Args:
@@ -157,25 +135,21 @@ def main(
Defaults to 1.0.
config_overrides: List of strings to override config values.
"""
assert config.eval is not None, "Eval config needs to be set"
assert config.eval.checkpoint_dir is not None, "Missing eval checkpoint dir: --eval.checkpoint-dir"
config_path = "../configs"
initialize(version_base="1.2", config_path=config_path)
config_overrides = config_overrides or []
config = compose(config_name, overrides=config_overrides)
config.trainer.resume_train.load_dir = checkpoint_dir
config.trainer.load_dir = config.eval.checkpoint_dir
config.pipeline.dataloader.eval_image_indices = None
config = setup_config(config) # converting to typed config
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# setup pipeline (which includes the dataloaders)
pipeline = setup_pipeline(config.pipeline, device=device, test_mode=True)
pipeline = config.pipeline.setup(device=device, test_mode=True)
pipeline.eval()
# load checkpointed information
checkpoint_name = _load_checkpoint(config.trainer.resume_train, pipeline)
checkpoint_name = _load_checkpoint(config.trainer, pipeline)
if method == MethodType.PSNR:
assert output_filename.endswith(".json")
if config.eval.method == cfg.MethodType.PSNR:
assert config.eval.output_filename.suffix == ".json"
stats_dict = render_stats_dict(pipeline)
avg_psnr = stats_dict["avg psnr"]
avg_rays_per_sec = stats_dict["avg rays per sec"]
@@ -184,36 +158,53 @@ def main(
print(f"Avg. Rays / Sec: {human_format(avg_rays_per_sec)}")
print(f"Avg. FPS: {avg_fps:0.4f}")
# save output to some file
io_utils.make_dir(output_filename)
config.eval.output_filename.parent.mkdir(parents=True, exist_ok=True)
benchmark_info = {
"config_name": config_name,
"checkpoint": checkpoint_name,
"experiment_name": config.experiment_name,
"method_name": config.method_name,
"checkpoint": str(checkpoint_name),
"results": stats_dict,
}
with open(output_filename, "w", encoding="utf8") as f:
json.dump(benchmark_info, f, indent=2)
print(f"Saved results to: {output_filename}")
with open(config.eval.output_filename, "w", encoding="utf8") as output_f:
json.dump(benchmark_info, output_f, indent=2)
print(f"Saved results to: {config.eval.output_filename}")
elif config.eval.method == cfg.MethodType.TRAJ:
assert config.eval.output_filename.suffix == ".mp4"
assert config.eval.rendered_output_name is not None, "Missing eval output type: --eval.rendered-output-name"
elif method == MethodType.TRAJ:
assert output_filename.endswith(".mp4")
# TODO(ethan): use camera information from parsing args
if traj == TrajectoryType.SPIRAL:
if config.eval.traj == cfg.TrajectoryType.SPIRAL:
camera_start = pipeline.dataloader.eval_dataloader.get_camera(image_idx=0)
# TODO(ethan): pass in the up direction of the camera
camera_path = get_spiral_path(camera_start, steps=30, radius=0.1)
elif traj == TrajectoryType.INTERP:
elif config.eval.traj == cfg.TrajectoryType.INTERP:
camera_start = pipeline.dataloader.eval_dataloader.get_camera(image_idx=0)
camera_end = pipeline.dataloader.eval_dataloader.get_camera(image_idx=10)
camera_path = get_interpolated_camera_path(camera_start, camera_end, steps=30)
else:
raise NotImplementedError
render_trajectory_video(
pipeline,
camera_path,
output_filename=output_filename,
rendered_output_name=rendered_output_name,
rendered_resolution_scaling_factor=rendered_resolution_scaling_factor,
output_filename=config.eval.output_filename,
rendered_output_name=config.eval.rendered_output_name,
rendered_resolution_scaling_factor=config.eval.rendered_resolution_scaling_factor,
num_rays_per_chunk=pipeline.dataloader.eval_num_rays_per_chunk,
)
if __name__ == "__main__":
dcargs.cli(main)
from nerfactory.configs.base_configs import base_configs
instantiated_config = cli_from_base_configs(base_configs, eval_mode=True)
if instantiated_config.eval.load_config:
logging.info(f"Loading pre-set config to: {instantiated_config.eval.load_config}")
with open(instantiated_config.eval.load_config, "rb") as config_f:
train_instantiated_config = pickle.load(config_f)
train_instantiated_config.eval = instantiated_config.eval
instantiated_config = train_instantiated_config
logging.info("Printing current config setup")
print(instantiated_config)
main(instantiated_config)
+31 -33
View File
@@ -3,25 +3,22 @@ run_train_nerf.py
"""
import logging
import os
import pickle
import random
import socket
import traceback
from datetime import timedelta
from typing import Any, Callable
from typing import Any, Callable, Optional
import hydra
import numpy as np
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
import yaml
from hydra.core.config_store import ConfigStore
from omegaconf import DictConfig
from nerfactory.configs import base as cfg
from nerfactory.configs.utils import cli_from_base_configs
from nerfactory.engine.trainer import train_loop
from nerfactory.utils import comms, profiler
from nerfactory.utils.config import Config, setup_config
logging.basicConfig(format="[%(filename)s:%(lineno)d] %(message)s", level=logging.DEBUG)
@@ -54,7 +51,7 @@ def _distributed_worker(
num_gpus_per_machine: int,
machine_rank: int,
dist_url: str,
config: Config,
config: cfg.Config,
timeout: timedelta = DEFAULT_TIMEOUT,
) -> Any:
"""Spawned distributed worker that handles the initialization of process group and handles the
@@ -89,8 +86,7 @@ def _distributed_worker(
timeout=timeout,
)
except Exception as e:
logger = logging.getLogger(__name__)
logger.error("Process group URL: %s", dist_url)
logging.error("Process group URL: %s", dist_url)
raise e
assert comms._LOCAL_PROCESS_GROUP is None # pylint: disable=protected-access
@@ -118,7 +114,7 @@ def launch(
num_machines: int = 1,
machine_rank: int = 0,
dist_url: str = "auto",
config: Config = None,
config: Optional[cfg.Config] = None,
timeout: timedelta = DEFAULT_TIMEOUT,
) -> None:
"""Function that spawns muliple processes to call on main_func
@@ -153,8 +149,7 @@ def launch(
port = _find_free_port()
dist_url = f"tcp://127.0.0.1:{port}"
if num_machines > 1 and dist_url.startswith("file://"):
logger = logging.getLogger(__name__)
logger.warning("file:// is not a reliable init_method in multi-machine jobs. Prefer tcp://")
logging.warning("file:// is not a reliable init_method in multi-machine jobs. Prefer tcp://")
process_context = mp.spawn(
_distributed_worker,
@@ -173,34 +168,18 @@ def launch(
try:
process_context.join()
except KeyboardInterrupt:
logger = logging.getLogger(__name__)
for i, process in enumerate(process_context.processes):
if process.is_alive():
logger.info("Terminating process %s", str(i))
logging.info("Terminating process %s", str(i))
process.terminate()
process.join()
logger.info("Process %s finished", str(i))
logging.info("Process %s finished", str(i))
finally:
profiler.flush_profiler(config.logging)
cs = ConfigStore.instance()
cs.store(name="graph_default", node=Config)
@hydra.main(version_base="1.2", config_path="../configs", config_name="graph_default.yaml")
def main(config: DictConfig):
def main(config: cfg.Config) -> None:
"""Main function."""
config = setup_config(config) # converting to typed config
unrolled_path = os.path.join(os.getcwd(), ".hydra/config.yaml")
if os.path.exists(unrolled_path):
with open(unrolled_path, encoding="utf8") as f:
unrolled_config = yaml.safe_load(f)
logger = logging.getLogger(__name__)
logger.info("Printing current config setup")
print(yaml.dump(unrolled_config, sort_keys=False, default_flow_style=False))
launch(
train_loop,
config.machine.num_gpus,
@@ -212,4 +191,23 @@ def main(config: DictConfig):
if __name__ == "__main__":
main() # pylint: disable=no-value-for-parameter
from nerfactory.configs.base_configs import base_configs
instantiated_config = cli_from_base_configs(base_configs)
if instantiated_config.trainer.load_config:
logging.info(f"Loading pre-set config to: {instantiated_config.trainer.load_config}")
with open(instantiated_config.trainer.load_config, "rb") as f:
instantiated_config = pickle.load(f)
# log and save config
logging.info("Printing current config setup")
print(instantiated_config)
config_pickle_path = instantiated_config.base_dir / "config.pkl"
config_json_path = instantiated_config.base_dir / "config.txt"
logging.info(f"Saving instance of config to: {config_pickle_path}")
logging.info(f"Saving human-readable config to: {config_json_path}")
config_pickle_path.parent.mkdir(parents=True, exist_ok=True)
with open(config_pickle_path, "wb") as f:
pickle.dump(instantiated_config, f)
with open(config_json_path, "w", encoding="utf8") as f:
f.write(str(instantiated_config))
main(config=instantiated_config)