From 71454a80b9dea72ee9acfa27a0ad4953cb519661 Mon Sep 17 00:00:00 2001 From: evonneng Date: Tue, 24 May 2022 19:12:40 +0000 Subject: [PATCH 1/5] update benchmarking for better practice --- scripts/benchmarking/launch_train_blender.sh | 11 ++++++++--- scripts/benchmarking/run_benchmark.py | 19 +++++++------------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/scripts/benchmarking/launch_train_blender.sh b/scripts/benchmarking/launch_train_blender.sh index 125d3c7cd..d523770c1 100755 --- a/scripts/benchmarking/launch_train_blender.sh +++ b/scripts/benchmarking/launch_train_blender.sh @@ -2,17 +2,22 @@ DATASETS=("mic" "ficus" "chair" "hotdog" "materials" "drums" "ship" "lego") GPU_IDX=1 + +date +now=$(date) +tag=$(date +'%m-%d-%Y') + for dataset in ${DATASETS[@]}; do export CUDA_VISIBLE_DEVICES=${!GPU_IDX} python scripts/run_train.py \ + '~logging.writer.LocalWriter' \ data.dataset.data_directory=data/blender/${dataset} \ data.dataset.downscale_factor=1 \ - experiment_name=blender_${dataset} \ + experiment_name=blender_${dataset}_${tag} \ graph.model_dir=mattport_models/ \ graph.steps_per_save=25000 \ graph.max_num_iterations=2000000 \ - logging.enable_stats=False \ logging.enable_profiler=False & - echo "Launched ${dataset} on gpu ${!GPU_IDX}" + echo "Launched ${dataset} on gpu ${!GPU_IDX}, ${tag}" GPU_IDX=$((GPU_IDX+1)) done \ No newline at end of file diff --git a/scripts/benchmarking/run_benchmark.py b/scripts/benchmarking/run_benchmark.py index 3b21f5715..2d26e77ec 100644 --- a/scripts/benchmarking/run_benchmark.py +++ b/scripts/benchmarking/run_benchmark.py @@ -17,16 +17,8 @@ from mattport.nerf.trainer import Trainer BENCH = { "method": "vanilla_nerf", "hydra_base_dir": "outputs/", - "ckpt_dir": { - "mic": "2022-05-12_173109", - "ficus": "2022-05-12_173109", - "chair": "2022-05-12_173109", - "hotdog": "2022-05-12_173109", - "materials": "2022-05-12_173109", - "drums": "2022-05-12_173109", - "ship": "2022-05-12_173109", - "lego": "2022-05-12_173109", - }, + "benchmark_date": "05-24-2022", + "object_list": ["mic", "ficus", "chair", "hotdog", "materials", "drums", "ship", "lego"], } @@ -48,8 +40,11 @@ def main(): benchmarks = {} hydra_base_dir = BENCH["hydra_base_dir"] method = BENCH["method"] - for dataset, ckpt_dir in tqdm(BENCH["ckpt_dir"].items()): - hydra_dir = f"{hydra_base_dir}/blender_{dataset}/{method}/{ckpt_dir}/" + benchmark_date = BENCH["benchmark_date"] + for dataset in tqdm(BENCH["object_list"]): + hydra_dir = f"{hydra_base_dir}/blender_{dataset}_{benchmark_date}/{method}/" + basename = os.listdir(hydra_dir)[0] + hydra_dir = f"{hydra_dir}/{basename}" initialize(config_path=os.path.join("../../", hydra_dir, ".hydra/")) config = compose("config.yaml") ckpt = load_best_ckpt(hydra_dir, config) From 1bf6d02900f55211afe75ea332071ab5ca0f307e Mon Sep 17 00:00:00 2001 From: evonneng Date: Tue, 24 May 2022 23:05:19 +0000 Subject: [PATCH 2/5] fixed logic for finding/distributing across GPUS --- scripts/benchmarking/launch_train_blender.sh | 34 +++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/scripts/benchmarking/launch_train_blender.sh b/scripts/benchmarking/launch_train_blender.sh index d523770c1..2355213f3 100755 --- a/scripts/benchmarking/launch_train_blender.sh +++ b/scripts/benchmarking/launch_train_blender.sh @@ -1,14 +1,39 @@ #!/bin/bash -DATASETS=("mic" "ficus" "chair" "hotdog" "materials" "drums" "ship" "lego") -GPU_IDX=1 +# Deal with gpu's. If passed in, use those. +GPU_IDX=("$@") +if [ -z "$GPU_IDX" ]; then + echo "no gpus set... finding available gpus" + # Find available devices + num_device=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) + START=0 + END=${num_device}-1 + GPU_IDX=() + for (( id=$START; id<$END; id++ )); do + free_mem=$(nvidia-smi --query-gpu=memory.free --format=csv -i $id | grep -Eo [0-9]+) + if [[ $free_mem -gt 10000 ]]; then + GPU_IDX+=( $id ) + fi + done +fi +echo available gpus... ${GPU_IDX[@]} + +DATASETS=("mic" "ficus" "chair" "hotdog" "materials" "drums" "ship" "lego") date now=$(date) tag=$(date +'%m-%d-%Y') +idx=0 +len=${#GPU_IDX[@]} +((len=len-1)) for dataset in ${DATASETS[@]}; do - export CUDA_VISIBLE_DEVICES=${!GPU_IDX} + export CUDA_VISIBLE_DEVICES=${GPU_IDX[$idx]} + if [ $idx == $len ]; then + idx=0 + else + ((idx=idx+1)) + fi python scripts/run_train.py \ '~logging.writer.LocalWriter' \ data.dataset.data_directory=data/blender/${dataset} \ @@ -18,6 +43,5 @@ for dataset in ${DATASETS[@]}; do graph.steps_per_save=25000 \ graph.max_num_iterations=2000000 \ logging.enable_profiler=False & - echo "Launched ${dataset} on gpu ${!GPU_IDX}, ${tag}" - GPU_IDX=$((GPU_IDX+1)) + echo "Launched ${dataset} on gpu ${GPU_IDX[$idx]}, ${tag}" done \ No newline at end of file From a782306c6b98e95e2661e6edee8a6fb64bc98c91 Mon Sep 17 00:00:00 2001 From: evonneng Date: Mon, 30 May 2022 15:56:28 +0000 Subject: [PATCH 3/5] added method to argument list --- scripts/benchmarking/launch_train_blender.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/benchmarking/launch_train_blender.sh b/scripts/benchmarking/launch_train_blender.sh index 2355213f3..e5b679539 100755 --- a/scripts/benchmarking/launch_train_blender.sh +++ b/scripts/benchmarking/launch_train_blender.sh @@ -1,5 +1,8 @@ #!/bin/bash +METHOD=$1 +shift + # Deal with gpu's. If passed in, use those. GPU_IDX=("$@") if [ -z "$GPU_IDX" ]; then @@ -35,6 +38,7 @@ for dataset in ${DATASETS[@]}; do ((idx=idx+1)) fi python scripts/run_train.py \ + --config-name ${METHOD} \ '~logging.writer.LocalWriter' \ data.dataset.data_directory=data/blender/${dataset} \ data.dataset.downscale_factor=1 \ @@ -43,5 +47,5 @@ for dataset in ${DATASETS[@]}; do graph.steps_per_save=25000 \ graph.max_num_iterations=2000000 \ logging.enable_profiler=False & - echo "Launched ${dataset} on gpu ${GPU_IDX[$idx]}, ${tag}" + echo "Launched ${METHOD} ${dataset} on gpu ${GPU_IDX[$idx]}, ${tag}" done \ No newline at end of file From 19109270feb9deec09ef08a606820b7a146e73b8 Mon Sep 17 00:00:00 2001 From: evonneng Date: Tue, 31 May 2022 21:50:26 +0000 Subject: [PATCH 4/5] update getting around readme --- README.md | 84 ++++++++++------------------------------ radiance/utils/README.md | 7 ++++ scripts/README.md | 52 +++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 64 deletions(-) create mode 100644 radiance/utils/README.md diff --git a/README.md b/README.md index e12ef89b9..761ddd1d1 100644 --- a/README.md +++ b/README.md @@ -52,75 +52,23 @@ python scripts/run_train.py data/dataset=friends_TBBT-big_living_room python scripts/run_train.py data/dataset=friends_TBBT-big_living_room graph.network.far_plane=14 ``` -# Logging and profiling features +# Getting around the codebase -We provide many logging functionalities for timing and/or tracking losses during training. All of these loggers are configurable via `configs/logging.yml` +The entry point for training starts at `scripts/run_train.py`, which spawns instances of our `Trainer()` class (in `nerf/trainer.py`). The `Trainer()` is responsible for setting up the datasets and NeRF graph depending on the config specified. If you are planning on just using our codebase to build a new NeRF method or use an existing implementation, we've abstracted away the training routine in these two files and chances are you will not need to touch them. -1. **Writer**: Logs losses and generated images during training to a specified output stream. Specify the type of writer (Tensorboard, Local Writer, Weights and Biases), and how often to log in the config. +The NeRF graph definitions can be found in `nerf/graph/`. Each implementation of NeRF is definined in its own file. For instance, `nerf/graph/instant_ngp.py` contains populates the `NGPGraph()` class with all of the appropriate fields, colliders, and misc. modules. +* Fields (`nerf/fields/`): composed of field modules (`nerf/field_modules/`) and represents the radiance field of the NeRF. +* Misc. Modules (`nerf/misc_modules`- TODO(maybe move to misc_modules? better organization)): any remaining module in the NeRF (e.g. renderers, samplers, losses, and metrics). -2. **Profiler**: Computes the average total time of execution for any function with the `@profiler.time_function` decorator. Prints out the full profile at the termination of training or the program. +To implement any pre-existing NeRF that we have not yet implemented under `nerf/graph/`, create a new graph structure by using provided modules or any new module you define. Then create an associated config with the appropriate parameters defined, with `__target__` pointing to your NeRF class. Then run training as described above. -# Setting up Jupyter -``` -python -m jupyter lab build -bash environments/run_jupyter.sh -``` +# Feature Documentation +## 1. [Hydra config structure](./configs/README.md) +## 2. [Logging, debugging utilities](./radiance/utils/README.md) +## 3. [Benchmarking, other tooling](./scripts/README.md) -# Tooling - -1. One can use [autoenv](https://github.com/hyperupcall/autoenv) to make setting up the environment and environment variables easier. This will run the `.envrc` file upon entering the `/path/to/radiance` folder. It will also remove the environment parameters upon leaving. - -``` -# Install direnv. -sudo apt install direnv - -# Add the following line to the bottom of your ~/.bashrc file. -eval "$(direnv hook bash)" - -# Populate your .envrc with commands you want to run. Then, run the following to allow updates. -cd /path/to/radiance -direnv allow . -``` - -2. To run local **github actions**, you can run: - -``` -python scripts/debugging/run_actions.py -``` - -3. To run local **profiling** to get a flame graph, make sure [pyspy](https://github.com/benfred/py-spy) is installed and you can run: - -``` -pip install py-spy - -## for flame graph -./scripts/debugging/profile.sh -t flame -o flame.svg -p scripts/run_train.py data/dataset=blender_lego - -## for live view of functions -./scripts/debugging/profile.sh -t top -p scripts/run_train.py data/dataset=blender_lego -``` - -4. For debugging with a debugger. - -``` -ipython --pdb scripts/run_train.py -``` - -5. **Benchmarking** For launching training jobs automatically on blender dataset - -``` -./scripts/benchmarking/launch_train_blender.sh ... -``` - -For testing specific methods, see scripts/benchmarking/run_benchmark.py. -Modify the `BENCH` variable to specify which jobs ("ckpt_dir") and methods ("method") you want benchmarked. Then run - -``` -python scripts/benchmarking/run_benchmark.py -``` - -# Running other repos with our data +## 4. Running other repos with our data ``` # nerf-pytorch @@ -133,7 +81,7 @@ conda activate jaxnerf python -m jaxnerf.train --data_dir=/path/to/radiance/data/blender/chair --train_dir=/path/to/radiance/outputs/blender_chair_jaxnerf --config=/path/to/radiance/external/jaxnerf/configs/demo --render_every 100 ``` -# Speeding up the code +## 5. Speeding up the code Documentation for running the code with CUDA. @@ -144,4 +92,12 @@ pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/to To run instant-ngp with tcnn, you can do the following. This is with the fox dataset. python scripts/run_train.py --config-name=instant_ngp_tcnn.yaml data/dataset=instant_ngp_fox +``` + + +## 6. Setting up Jupyter + +``` +python -m jupyter lab build +bash environments/run_jupyter.sh ``` \ No newline at end of file diff --git a/radiance/utils/README.md b/radiance/utils/README.md new file mode 100644 index 000000000..e47d4e76a --- /dev/null +++ b/radiance/utils/README.md @@ -0,0 +1,7 @@ +# Logging and profiling features + +We provide many logging functionalities for timing and/or tracking losses during training. All of these loggers are configurable via `configs/logging.yml` + +1. **Writer**: Logs losses and generated images during training to a specified output stream. Specify the type of writer (Tensorboard, Local Writer, Weights and Biases), and how often to log in the config. + +2. **Profiler**: Computes the average total time of execution for any function with the `@profiler.time_function` decorator. Prints out the full profile at the termination of training or the program. \ No newline at end of file diff --git a/scripts/README.md b/scripts/README.md index e69de29bb..acfba3558 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -0,0 +1,52 @@ +# Tooling + +1. One can use [autoenv](https://github.com/hyperupcall/autoenv) to make setting up the environment and environment variables easier. This will run the `.envrc` file upon entering the `/path/to/radiance` folder. It will also remove the environment parameters upon leaving. + +``` +# Install direnv. +sudo apt install direnv + +# Add the following line to the bottom of your ~/.bashrc file. +eval "$(direnv hook bash)" + +# Populate your .envrc with commands you want to run. Then, run the following to allow updates. +cd /path/to/radiance +direnv allow . +``` + +2. To run local **github actions**, you can run: + +``` +python scripts/debugging/run_actions.py +``` + +3. To run local **profiling** to get a flame graph, make sure [pyspy](https://github.com/benfred/py-spy) is installed and you can run: + +``` +pip install py-spy + +## for flame graph +./scripts/debugging/profile.sh -t flame -o flame.svg -p scripts/run_train.py data/dataset=blender_lego + +## for live view of functions +./scripts/debugging/profile.sh -t top -p scripts/run_train.py data/dataset=blender_lego +``` + +4. For debugging with a debugger. + +``` +ipython --pdb scripts/run_train.py +``` + +5. **Benchmarking** For launching training jobs automatically on blender dataset + +``` +./scripts/benchmarking/launch_train_blender.sh ... +``` + +For testing specific methods, see scripts/benchmarking/run_benchmark.py. +Modify the `BENCH` variable to specify which jobs ("ckpt_dir") and methods ("method") you want benchmarked. Then run + +``` +python scripts/benchmarking/run_benchmark.py +``` \ No newline at end of file From 05a086e24f92959522979c1ec83755c171648e82 Mon Sep 17 00:00:00 2001 From: evonneng Date: Tue, 31 May 2022 22:02:25 +0000 Subject: [PATCH 5/5] restruct --- README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 761ddd1d1..7a478a84f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Radiance +# Radiance :bulb: # Quickstart @@ -60,15 +60,15 @@ The NeRF graph definitions can be found in `nerf/graph/`. Each implementation of * Fields (`nerf/fields/`): composed of field modules (`nerf/field_modules/`) and represents the radiance field of the NeRF. * Misc. Modules (`nerf/misc_modules`- TODO(maybe move to misc_modules? better organization)): any remaining module in the NeRF (e.g. renderers, samplers, losses, and metrics). -To implement any pre-existing NeRF that we have not yet implemented under `nerf/graph/`, create a new graph structure by using provided modules or any new module you define. Then create an associated config with the appropriate parameters defined, with `__target__` pointing to your NeRF class. Then run training as described above. +To implement any pre-existing NeRF that we have not yet implemented under `nerf/graph/`, create a new graph structure by using provided modules or any new module you define. Then create an associated config making sure `__target__` points to your NeRF class (see [here](./configs/README.md) for more info on how to create the config). Then run training as described above. # Feature Documentation -## 1. [Hydra config structure](./configs/README.md) -## 2. [Logging, debugging utilities](./radiance/utils/README.md) -## 3. [Benchmarking, other tooling](./scripts/README.md) +### 1. [Hydra config structure](./configs/README.md) +### 2. [Logging, debugging utilities](./radiance/utils/README.md) +### 3. [Benchmarking, other tooling](./scripts/README.md) -## 4. Running other repos with our data +### 4. Running other repos with our data ``` # nerf-pytorch @@ -81,21 +81,21 @@ conda activate jaxnerf python -m jaxnerf.train --data_dir=/path/to/radiance/data/blender/chair --train_dir=/path/to/radiance/outputs/blender_chair_jaxnerf --config=/path/to/radiance/external/jaxnerf/configs/demo --render_every 100 ``` -## 5. Speeding up the code - +### 5. Speeding up the code Documentation for running the code with CUDA. +Please see https://github.com/NVlabs/tiny-cuda-nn for how to install tiny-cuda-nn. ``` -This is how to install tiny-cuda-nn. -# https://github.com/NVlabs/tiny-cuda-nn pip install git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch +``` To run instant-ngp with tcnn, you can do the following. This is with the fox dataset. +``` python scripts/run_train.py --config-name=instant_ngp_tcnn.yaml data/dataset=instant_ngp_fox ``` -## 6. Setting up Jupyter +### 6. Setting up Jupyter ``` python -m jupyter lab build