4
0
mirror of https://github.com/nerfstudio-project/nerfstudio.git synced 2026-11-11 02:24:38 +08:00
nerfstudio/tests/test_train.py
Terrance Wang b17268e83b
[WIP] dreamfusion implementation (#1008)
* starting the framework for dreamfusion

* Removed view direction from RGB to calculate albedo

* Started diffusion

* Added dreamfusion to CLI, removed direction embeddings

* Normals in dreamfusion

* Lambertian shading added

* Train normals in Dreamfusion

* Stable diffusion from Stable Dreamfusion

* Fix normal orientation loss

* Configurable light direction sampling

* adding density initalization at origin

* V1 datamanager

* Don't train normals for first 1k iters

* SDS loss added to dreamfusion

* Dreamfusion pipeline

* Background MLP and SphereCollider

* Normals prediction training fixed

* Sigmoid on background, prompt in config

* Shading hyperparameter adjustements

* Shading hyperparameter tuning

* Location based prompting, density initialization fades, LR change

* Cleaned imports, more config options

* Tuning, more config changes

* remove unused sampler, normalize predicted normals from network

* change position encoding to direction encoding for background color mlp

* Updating dreamfusion to use custom trainer, fixed bug, multi-gpu support (put diffusion on one specific device), merged with main

* Hard coding less, offloading to config

* Appeasing the linter

* First pass opacity / sparsity losses

* Small patches and configs

* Accidentally pushed logs

* Removing logs for real this time

* Transmittance loss bug fix

* 12 GB Dreamfusion!

* Lint

* Remove unecissary collider changes

* Start SD cleanup

* Cleanup SD code

* adding overhead location prompting

* Use lambertian renderer

* turn grad accum off by default, decrease background mlp size, add overhead text embedding

* Cleanup

* Fix normals vis

* change uniform sampler to prediction network, add option to jit unet for faster sds_loss

* Use shaders

* cleaning up jit code

* add warning if using non jit unet

* add proposal network optimizer to config

* speed up render and cleanup code

* Clean up datamanager

* Add distortion loss

* fix wanb

* adding interpolated location sampling

* Update configs

* interpolated location prompting option, lowering init density, transmittance loss interpolation flipped

* rename alphas mult to alphas loss mult

* Mess around with normals

* add option to use random bg color + bg mlp

* Remove Dreamfusion Trainer (#1423)

* Dreamfusion trainer no longer needed

* Added typing for grad scaler

* SDS return loss

* Fix CFG

* Cleanup

* Update configs

* Fix opacity loss, use unform sampling first round of prop

* add prompt edditing to viewer

* Choose your own diffusion

* remove dreamfusion pipeline file

* merging + remove dreamfusion pipeline file

* Update configs

* add stable diff utils

* cleanup

* cleanup

* Fix test

* positional prompting bugfix

* Warmups

* tune

* lint

* Remove dataparser

* merging + text embedding bugfix

* Set data path to launch viewer for DreamFusion (#1776)

* Set data path to launch viewer for DreamFusion

* Apply formatter

* Fallback to 'base_dir' if data manager has no path

* Make if statement non-inline

* Reformat

---------

Co-authored-by: Matthew Tancik <mattt4001@gmail.com>

* add deepfloyd option first try

* merging

* merging from main

* merging

* removing comments, cleaning code, making stable diff utils usable for deepfloyd, remove deepfloyd text encoder after use

* undo chagnes in data folder

* cleaning code, removing unneccesary changes from main, removed normals prediction network

* remove old scripts files, rename generative pipeline flag to no_input_data, revert to default histogram padding in proposal sampler

* changing pipeline flag

* move no input data flag from pipeline to datamanager

* undo config and merge in deepfloyd from main

* merging grad scaler pipeline addition

* merging + setting proposal net scheduler

* revert nerfacto config change

* typing + linting

* rename dreamfusion datamanager to random camras datamanager, check datamanager typing in pipeline and trainer

* fix test import

* fix test import

* changing datamanager check in trainer, add typing in random cameras datamanager

* add Tensor to randomcamerasdatamanager

* remove old comments

* resolve tyro warning

* update fields to use tcnn

* lint

* add packages for deepfloyd

* add versions

* docs first draft

* rename to generfacto, clean docs

* fix import error

* minor changes

* fix orientation loss bug, retune params

* cleanup

* typing

* trying docs fix

* fixing generative package imports

* fix pyright

* docs fix

* merge w main

* docs minor change

* tuning + bugfixes

* remove torch.backends

---------

Co-authored-by: Abhik Ahuja <ahujaabhik@berkeley.edu>
Co-authored-by: David McAllister <davidrmc2@gmail.com>
Co-authored-by: Jake Austin <jake-austin@berkeley.edu>
Co-authored-by: Matthew Tancik <tancik@berkeley.edu>
Co-authored-by: Matt Tancik <mattt4001@gmail.com>
Co-authored-by: SeungWoo Yoo <dreamy1534@gmail.com>
Co-authored-by: ayaanzhaque <ayaanzhaque@gmail.com>
Co-authored-by: Ayaan Haque <ayaanzhaque@berkeley.edu>
2023-06-17 23:34:04 -07:00

89 lines
2.9 KiB
Python

"""
Default test to make sure train runs
"""
from __future__ import annotations
from pathlib import Path
import pytest
from nerfstudio.configs.method_configs import method_configs
from nerfstudio.data.dataparsers.blender_dataparser import BlenderDataParserConfig
from nerfstudio.data.dataparsers.minimal_dataparser import MinimalDataParserConfig
from nerfstudio.engine.trainer import TrainerConfig
from nerfstudio.models.vanilla_nerf import VanillaModelConfig
from nerfstudio.scripts.train import train_loop
BLACKLIST = [
"base",
"semantic-nerfw",
"instant-ngp",
"instant-ngp-bounded",
"nerfacto-big",
"volinga",
"phototourism",
"depth-nerfacto",
"neus",
"generfacto",
"neus-facto",
]
def set_reduced_config(config: TrainerConfig):
"""Reducing the config settings to speedup test"""
config.machine.device_type = "cpu"
if hasattr(config.pipeline.model, "implementation"):
setattr(config.pipeline.model, "implementation", "torch")
config.mixed_precision = False
config.use_grad_scaler = False
config.max_num_iterations = 2
# reduce dataset factors; set dataset to test
config.pipeline.datamanager.dataparser = BlenderDataParserConfig(data=Path("tests/data/lego_test"))
config.pipeline.datamanager.train_num_images_to_sample_from = 1
config.pipeline.datamanager.train_num_rays_per_batch = 4
# use tensorboard logging instead of wandb
config.vis = "tensorboard"
config.logging.relative_log_dir = Path("/tmp/")
# reduce model factors
if hasattr(config.pipeline.model, "num_coarse_samples"):
assert isinstance(config.pipeline.model, VanillaModelConfig)
config.pipeline.model.num_coarse_samples = 4
if hasattr(config.pipeline.model, "num_importance_samples"):
assert isinstance(config.pipeline.model, VanillaModelConfig)
config.pipeline.model.num_importance_samples = 4
# remove viewer
config.viewer.quit_on_train_completion = True
return config
@pytest.mark.filterwarnings("ignore::DeprecationWarning")
def test_train():
"""test run train script works properly"""
all_config_names = method_configs.keys()
for config_name in all_config_names:
if config_name in BLACKLIST:
print("skipping", config_name)
continue
print(f"testing run for: {config_name}")
config = method_configs[config_name]
config = set_reduced_config(config)
train_loop(local_rank=0, world_size=0, config=config)
def test_simple_io():
"""test to check minimal data IO works correctly"""
config = method_configs["vanilla-nerf"]
config.pipeline.datamanager.dataparser = MinimalDataParserConfig(data=Path("tests/data/minimal_parser"))
config = set_reduced_config(config)
train_loop(local_rank=0, world_size=0, config=config)
if __name__ == "__main__":
test_train()
test_simple_io()