Files
nerfstudio/pyproject.toml
T
b17268e83b [WIP] dreamfusion implementation (#1008)
* starting the framework for dreamfusion

* Removed view direction from RGB to calculate albedo

* Started diffusion

* Added dreamfusion to CLI, removed direction embeddings

* Normals in dreamfusion

* Lambertian shading added

* Train normals in Dreamfusion

* Stable diffusion from Stable Dreamfusion

* Fix normal orientation loss

* Configurable light direction sampling

* adding density initalization at origin

* V1 datamanager

* Don't train normals for first 1k iters

* SDS loss added to dreamfusion

* Dreamfusion pipeline

* Background MLP and SphereCollider

* Normals prediction training fixed

* Sigmoid on background, prompt in config

* Shading hyperparameter adjustements

* Shading hyperparameter tuning

* Location based prompting, density initialization fades, LR change

* Cleaned imports, more config options

* Tuning, more config changes

* remove unused sampler, normalize predicted normals from network

* change position encoding to direction encoding for background color mlp

* Updating dreamfusion to use custom trainer, fixed bug, multi-gpu support (put diffusion on one specific device), merged with main

* Hard coding less, offloading to config

* Appeasing the linter

* First pass opacity / sparsity losses

* Small patches and configs

* Accidentally pushed logs

* Removing logs for real this time

* Transmittance loss bug fix

* 12 GB Dreamfusion!

* Lint

* Remove unecissary collider changes

* Start SD cleanup

* Cleanup SD code

* adding overhead location prompting

* Use lambertian renderer

* turn grad accum off by default, decrease background mlp size, add overhead text embedding

* Cleanup

* Fix normals vis

* change uniform sampler to prediction network, add option to jit unet for faster sds_loss

* Use shaders

* cleaning up jit code

* add warning if using non jit unet

* add proposal network optimizer to config

* speed up render and cleanup code

* Clean up datamanager

* Add distortion loss

* fix wanb

* adding interpolated location sampling

* Update configs

* interpolated location prompting option, lowering init density, transmittance loss interpolation flipped

* rename alphas mult to alphas loss mult

* Mess around with normals

* add option to use random bg color + bg mlp

* Remove Dreamfusion Trainer (#1423)

* Dreamfusion trainer no longer needed

* Added typing for grad scaler

* SDS return loss

* Fix CFG

* Cleanup

* Update configs

* Fix opacity loss, use unform sampling first round of prop

* add prompt edditing to viewer

* Choose your own diffusion

* remove dreamfusion pipeline file

* merging + remove dreamfusion pipeline file

* Update configs

* add stable diff utils

* cleanup

* cleanup

* Fix test

* positional prompting bugfix

* Warmups

* tune

* lint

* Remove dataparser

* merging + text embedding bugfix

* Set data path to launch viewer for DreamFusion (#1776)

* Set data path to launch viewer for DreamFusion

* Apply formatter

* Fallback to 'base_dir' if data manager has no path

* Make if statement non-inline

* Reformat

---------

Co-authored-by: Matthew Tancik <mattt4001@gmail.com>

* add deepfloyd option first try

* merging

* merging from main

* merging

* removing comments, cleaning code, making stable diff utils usable for deepfloyd, remove deepfloyd text encoder after use

* undo chagnes in data folder

* cleaning code, removing unneccesary changes from main, removed normals prediction network

* remove old scripts files, rename generative pipeline flag to no_input_data, revert to default histogram padding in proposal sampler

* changing pipeline flag

* move no input data flag from pipeline to datamanager

* undo config and merge in deepfloyd from main

* merging grad scaler pipeline addition

* merging + setting proposal net scheduler

* revert nerfacto config change

* typing + linting

* rename dreamfusion datamanager to random camras datamanager, check datamanager typing in pipeline and trainer

* fix test import

* fix test import

* changing datamanager check in trainer, add typing in random cameras datamanager

* add Tensor to randomcamerasdatamanager

* remove old comments

* resolve tyro warning

* update fields to use tcnn

* lint

* add packages for deepfloyd

* add versions

* docs first draft

* rename to generfacto, clean docs

* fix import error

* minor changes

* fix orientation loss bug, retune params

* cleanup

* typing

* trying docs fix

* fixing generative package imports

* fix pyright

* docs fix

* merge w main

* docs minor change

* tuning + bugfixes

* remove torch.backends

---------

Co-authored-by: Abhik Ahuja <ahujaabhik@berkeley.edu>
Co-authored-by: David McAllister <davidrmc2@gmail.com>
Co-authored-by: Jake Austin <jake-austin@berkeley.edu>
Co-authored-by: Matthew Tancik <tancik@berkeley.edu>
Co-authored-by: Matt Tancik <mattt4001@gmail.com>
Co-authored-by: SeungWoo Yoo <dreamy1534@gmail.com>
Co-authored-by: ayaanzhaque <ayaanzhaque@gmail.com>
Co-authored-by: Ayaan Haque <ayaanzhaque@berkeley.edu>
2023-06-17 23:34:04 -07:00

182 lines
4.9 KiB
TOML

[build-system]
requires = ["setuptools>=61.0"]
build-backend = "setuptools.build_meta"
[project]
name = "nerfstudio"
version = "0.3.1"
description = "All-in-one repository for state-of-the-art NeRFs"
readme = "README.md"
license = { text="Apache 2.0"}
requires-python = ">=3.8.0"
classifiers = [
"Development Status :: 3 - Alpha",
"Programming Language :: Python",
]
dependencies = [
"appdirs>=1.4",
"av>=9.2.0",
"cryptography>=38",
"tyro>=0.5.3",
"gdown>=4.6.0",
"ninja>=1.10",
"h5py>=2.9.0",
"imageio>=2.21.1",
'importlib-metadata>=6.0.0; python_version < "3.10"',
"ipywidgets>=7.6",
"jaxtyping>=0.2.15",
"jupyterlab>=3.3.4",
"matplotlib>=3.5.3",
"mediapy>=1.1.0",
"msgpack>=1.0.4",
"msgpack_numpy>=0.4.8",
"nerfacc==0.5.2",
"open3d>=0.16.0",
"opencv-python==4.6.0.66",
"Pillow>=9.3.0",
"plotly>=5.7.0",
"protobuf<=3.20.3,!=3.20.0",
# TODO(1480) enable when pycolmap windows wheels are available
# "pycolmap==0.3.0",
"pyequilib>=0.5.6",
"pymeshlab>=2022.2.post2",
"pyngrok>=5.1.0",
"python-socketio>=5.7.1",
"pyquaternion>=0.9.9",
"requests",
"rich>=12.5.1",
"scikit-image>=0.19.3",
"tensorboard>=2.13.0",
"torch>=1.13.1",
"torchvision>=0.14.1",
"torchmetrics[image]>=0.9.3",
"typing_extensions>=4.4.0",
"viser>=0.0.12",
"nuscenes-devkit>=1.1.1",
"wandb>=0.13.3",
"xatlas",
"trimesh>=3.20.2"
]
[project.urls]
"Documentation" = "https://docs.nerf.studio"
[project.optional-dependencies]
# Generative related dependencies
gen = [
"diffusers==0.16.1",
"transformers==4.29.2",
"accelerate==0.19.0",
"bitsandbytes==0.39.0",
"sentencepiece==0.1.99",
]
# Development packages
dev = [
"black[jupyter]==23.3.0",
"pre-commit==3.3.2",
"pytest==7.1.2",
"pytest-xdist==2.5.0",
"typeguard==2.13.3",
"ruff==0.0.267",
"sshconf==0.2.5",
"pycolmap==0.3.0",
"diffusers==0.16.1",
"opencv-stubs==0.0.7",
"transformers==4.29.2",
"pyright==1.1.308",
]
# Documentation related packages
docs = [
"furo==2022.09.29",
# Specifying ipython for https://github.com/ipython/ipython/issues/13845
"ipython==8.6.0",
"readthedocs-sphinx-search==0.1.2",
"myst-nb==0.16.0",
"nbconvert==7.2.5",
"nbformat==5.5.0",
"sphinx==5.2.1",
"sphinxemoji==0.2.0",
"sphinx-argparse==0.3.1",
"sphinx-copybutton==0.5.0",
"sphinx-design==0.2.0",
"sphinxext-opengraph==0.6.3"
]
[project.scripts]
# Note, add entrypoint name to nerfstudio/scripts/completions/install.py to include CLI completion
ns-install-cli = "nerfstudio.scripts.completions.install:entrypoint"
ns-process-data = "nerfstudio.scripts.process_data:entrypoint"
ns-download-data = "nerfstudio.scripts.downloads.download_data:entrypoint"
ns-train = "nerfstudio.scripts.train:entrypoint"
ns-viewer = "nerfstudio.scripts.viewer.run_viewer:entrypoint"
ns-eval = "nerfstudio.scripts.eval:entrypoint"
ns-render = "nerfstudio.scripts.render:entrypoint"
ns-export = "nerfstudio.scripts.exporter:entrypoint"
ns-dev-test = "nerfstudio.scripts.github.run_actions:entrypoint"
ns-dev-sync-viser-message-defs = "nerfstudio.scripts.viewer.sync_viser_message_defs:entrypoint"
[options]
# equivalent to using --extra-index-url with pip, which is needed for specifying the CUDA version torch and torchvision
dependency_links = [
"https://download.pytorch.org/whl/cu118"
]
[tool.setuptools.packages.find]
include = ["nerfstudio*"]
[tool.setuptools.package-data]
"*" = ["*.cu", "*.json", "py.typed", "setup.bash", "setup.zsh"]
[tool.black]
line-length = 120
[tool.pytest.ini_options]
addopts = "-n=4 --typeguard-packages=nerfstudio --disable-warnings"
testpaths = [
"tests",
]
[tool.pyright]
include = ["nerfstudio"]
exclude = ["**/node_modules",
"**/__pycache__",
]
defineConstant = { DEBUG = true }
reportMissingImports = "warning"
reportMissingTypeStubs = false
reportPrivateImportUsage = false
pythonVersion = "3.8"
pythonPlatform = "Linux"
[tool.ruff]
select = [
"E", # pycodestyle errors.
"F", # Pyflakes rules.
"PLC", # Pylint convention warnings.
"PLE", # Pylint errors.
"PLR", # Pylint refactor recommendations.
"PLW", # Pylint warnings.
]
ignore = [
"E501", # Line too long.
"F722", # Forward annotation false positive from jaxtyping. Should be caught by pyright.
"F821", # Forward annotation false positive from jaxtyping. Should be caught by pyright.
"PLR2004", # Magic value used in comparison.
"PLR0915", # Too many statements.
"PLR0913", # Too many arguments.
"PLC0414", # Import alias does not rename variable. (this is used for exporting names)
"PLC1901", # Use falsey strings.
"PLR5501", # Use `elif` instead of `else if`.
"PLR0911", # Too many return statements.
"PLR0912", # Too many branches.
"PLW0603", # Globa statement updates are discouraged.
"PLW2901", # For loop variable overwritten.
]