diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 84f99bf49..f6057fdd1 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -8,6 +8,7 @@ "mutantdino.resourcemonitor", "ms-azuretools.vscode-docker", "nvidia.nsight-vscode-edition", - "ms-python.python" + "ms-python.python", + "charliermarsh.ruff" ] -} \ No newline at end of file +} diff --git a/.github/workflows/core_code_checks.yml b/.github/workflows/core_code_checks.yml index 3e30fba7d..761d4841b 100644 --- a/.github/workflows/core_code_checks.yml +++ b/.github/workflows/core_code_checks.yml @@ -32,13 +32,13 @@ jobs: - name: Check notebook cell metadata run: | python ./nerfstudio/scripts/docs/add_nb_tags.py --check - - name: Run isort - run: isort docs/ nerfstudio/ tests/ --profile black --check + - name: Run Ruff + run: ruff docs/ nerfstudio/ tests/ - name: Run Black run: black docs/ nerfstudio/ tests/ --check - - name: Python Pylint + - name: Run Pyright run: | - pylint nerfstudio tests + pyright - name: Test with pytest run: | pytest diff --git a/.gitignore b/.gitignore index 7f1688d59..873fe16d9 100644 --- a/.gitignore +++ b/.gitignore @@ -169,9 +169,7 @@ renders/ events.out.* # Data -data -!*/data -!docs/reference/api/data/ +/data # Misc old/ diff --git a/.vscode/settings.json b/.vscode/settings.json index 90f5ae01c..a02b23dd9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -26,17 +26,17 @@ "[python]": { "editor.defaultFormatter": "ms-python.python", "editor.codeActionsOnSave": { - "source.organizeImports": true + "source.organizeImports": true, + "source.fixAll": true } }, "editor.formatOnSave": true, "python.envFile": "${workspaceFolder}/.env", "python.formatting.provider": "black", "python.formatting.blackArgs": ["--line-length=120"], - "python.linting.pylintEnabled": true, + "python.linting.pylintEnabled": false, "python.linting.flake8Enabled": false, "python.linting.enabled": true, - "python.sortImports.args": ["--src=${workspaceFolder}", "--profile", "black"], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, "python.linting.ignorePatterns": [ @@ -122,5 +122,5 @@ }, "python.analysis.typeCheckingMode": "basic", "python.analysis.diagnosticMode": "workspace", - "eslint.packageManager": "yarn" + "eslint.packageManager": "yarn", } diff --git a/Dockerfile b/Dockerfile index 92af70932..755b67c52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -114,8 +114,8 @@ SHELL ["/bin/bash", "-c"] # Upgrade pip and install packages. RUN python3.10 -m pip install --upgrade pip setuptools pathtools promise pybind11 -# Install pytorch and submodules (Currently, we still use cu116 which is the latest version for torch 1.12.1 and is compatible with CUDA 11.8). -RUN python3.10 -m pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116 +# Install pytorch and submodules +RUN python3.10 -m pip install torch==2.0.1+cu118 torchvision==0.15.2+cu118 --extra-index-url https://download.pytorch.org/whl/cu118 # Install tynyCUDNN (we need to set the target architectures as environment variable first). ENV TCNN_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES} RUN python3.10 -m pip install git+https://github.com/NVlabs/tiny-cuda-nn.git@v1.6#subdirectory=bindings/torch diff --git a/README.md b/README.md index 15f8dfa46..ef9b1ed89 100644 --- a/README.md +++ b/README.md @@ -70,20 +70,20 @@ Nerfstudio initially launched as an opensource project by Berkeley students in [ We are committed to providing learning resources to help you understand the basics of (if you're just getting started), and keep up-to-date with (if you're a seasoned veteran) all things NeRF. As researchers, we know just how hard it is to get onboarded with this next-gen technology. So we're here to help with tutorials, documentation, and more! -Have feature requests? Want to add your brand-spankin'-new NeRF model? Have a new dataset? **We welcome any and all [contributions](https://docs.nerf.studio/en/latest/reference/contributing.html)!** Please do not hesitate to reach out to the nerfstudio team with any questions via [Discord](https://discord.gg/uMbNqcraFc). +Have feature requests? Want to add your brand-spankin'-new NeRF model? Have a new dataset? **We welcome [contributions](https://docs.nerf.studio/en/latest/reference/contributing.html)!** Please do not hesitate to reach out to the nerfstudio team with any questions via [Discord](https://discord.gg/uMbNqcraFc). We hope nerfstudio enables you to build faster :hammer: learn together :books: and contribute to our NeRF community :sparkling_heart:. # Quickstart The quickstart will help you get started with the default vanilla NeRF trained on the classic Blender Lego scene. -For more complex changes (e.g., running with your own data/setting up a new NeRF graph, please refer to our [references](#learn-more). +For more complex changes (e.g., running with your own data/setting up a new NeRF graph), please refer to our [references](#learn-more). ## 1. Installation: Setup the environment ### Prerequisites -You must have an NVIDIA video card with CUDA installed on the system. This library has been tested with version 11.3 of CUDA. You can find more information about installing CUDA [here](https://docs.nvidia.com/cuda/cuda-quick-start-guide/index.html) +You must have an NVIDIA video card with CUDA installed on the system. This library has been tested with version 11.8 of CUDA. You can find more information about installing CUDA [here](https://docs.nvidia.com/cuda/cuda-quick-start-guide/index.html) ### Create environment @@ -97,19 +97,19 @@ python -m pip install --upgrade pip ### Dependencies -Install pytorch with CUDA (this repo has been tested with CUDA 11.3 and CUDA 11.7) and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn) - -For CUDA 11.3: - -```bash -pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 -f https://download.pytorch.org/whl/torch_stable.html -pip install ninja git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch -``` +Install pytorch with CUDA (this repo has been tested with CUDA 11.7 and CUDA 11.8) and [tiny-cuda-nn](https://github.com/NVlabs/tiny-cuda-nn) For CUDA 11.7: ```bash -pip install torch==1.13.1 torchvision functorch --extra-index-url https://download.pytorch.org/whl/cu117 +pip install torch==2.0.1+cu117 torchvision==0.15.2+cu117 --extra-index-url https://download.pytorch.org/whl/cu117 +pip install ninja git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch +``` + +For CUDA 11.8: + +```bash +pip install torch==2.0.1+cu118 torchvision==0.15.2+cu118 --extra-index-url https://download.pytorch.org/whl/cu118 pip install ninja git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch ``` @@ -317,13 +317,17 @@ You can find a paper writeup of the framework on [arXiv](https://arxiv.org/abs/2 If you use this library or find the documentation useful for your research, please consider citing: ``` -@article{nerfstudio, - author = {Tancik, Matthew and Weber, Ethan and Ng, Evonne and Li, Ruilong and Yi, - Brent and Kerr, Justin and Wang, Terrance and Kristoffersen, Alexander and Austin, - Jake and Salahi, Kamyar and Ahuja, Abhik and McAllister, David and Kanazawa, Angjoo}, - title = {Nerfstudio: A Modular Framework for Neural Radiance Field Development}, - journal = {arXiv preprint arXiv:2302.04264}, - year = {2023}, +@inproceedings{nerfstudio, + title = {Nerfstudio: A Modular Framework for Neural Radiance Field Development}, + author = { + Tancik, Matthew and Weber, Ethan and Ng, Evonne and Li, Ruilong and Yi, Brent + and Kerr, Justin and Wang, Terrance and Kristoffersen, Alexander and Austin, + Jake and Salahi, Kamyar and Ahuja, Abhik and McAllister, David and Kanazawa, + Angjoo + }, + year = 2023, + booktitle = {ACM SIGGRAPH 2023 Conference Proceedings}, + series = {SIGGRAPH '23} } ``` diff --git a/colab/demo.ipynb b/colab/demo.ipynb index 541b9d741..f5c9e29e9 100644 --- a/colab/demo.ipynb +++ b/colab/demo.ipynb @@ -343,7 +343,7 @@ "camera_path_filename = camera_path_filename.replace(\" \", \"\\\\ \").replace(\"(\", \"\\\\(\").replace(\")\", \"\\\\)\")\n", "\n", "%cd /content/\n", - "!ns-render --load-config $config_filename --traj filename --camera-path-filename $camera_path_filename --output-path renders/output.mp4" + "!ns-render camera-path --load-config $config_filename --camera-path-filename $camera_path_filename --output-path renders/output.mp4" ] } ], @@ -371,4 +371,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} diff --git a/docs/conf.py b/docs/conf.py index 824a61d78..be838235d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,3 @@ -# pylint: skip-file # Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full @@ -16,7 +15,6 @@ import sys sys.path.insert(0, os.path.abspath("..")) sys.path.append(os.path.abspath("./_pygments")) -import nerfstudio.configs.base_config # -- Project information ----------------------------------------------------- diff --git a/docs/developer_guides/new_methods.md b/docs/developer_guides/new_methods.md index baafad440..f0b3a51d7 100644 --- a/docs/developer_guides/new_methods.md +++ b/docs/developer_guides/new_methods.md @@ -16,6 +16,7 @@ We recommend the following file structure: │ ├── custom_model.py [optional] │ ├── custom_field.py [optional] │ ├── custom_datamanger.py [optional] +│ ├── custom_dataparser.py [optional] │ ├── ... ├── pyproject.toml ``` @@ -72,10 +73,44 @@ pip install -e . When developing a new method you don't always want to install your code as a package. Instead, you may use the `NERFSTUDIO_METHOD_CONFIGS` environment variable to temporarily register your custom method. + ``` export NERFSTUDIO_METHOD_CONFIGS="my-method=my_method.my_config:MyMethod" ``` +## Registering custom dataparser with nerfstudio + +We also support adding new dataparsers in a similar way. In order to extend the NeRFstudio and register a customized dataparser, you can register it with Nerfstudio as a `nerfstudio.dataparser_configs` entrypoint in the `pyproject.toml` file. Nerfstudio will automatically look for all registered dataparsers and will register them to be used by methods such as `ns-train`. + +You can declare the dataparser in the same config file: + +```python +"""my_method/my_config.py""" + +from nerfstudio.plugins.registry_dataparser import DataParserSpecification +from my_method.custom_dataparser import CustomDataparserConfig + +MyDataparser = DataParserSpecification(config=CustomDataparserConfig()) +``` + +Then add the following lines in the `pyproject.toml` file, where the entrypoint to the new dataparser is set. + +```python +"""pyproject.toml""" + +[project] +name = "my_method" + +[project.entry-points.'nerfstudio.dataparser_configs'] +custom-dataparser = 'my_method.my_config:MyDataparser' +``` + +finally run the following to register the dataparser. + +``` +pip install -e . +``` + ## Running custom method After registering your method you should be able to run the method with, diff --git a/docs/developer_guides/pipelines/dataparsers.md b/docs/developer_guides/pipelines/dataparsers.md index 19b964e5a..cb1693aa7 100644 --- a/docs/developer_guides/pipelines/dataparsers.md +++ b/docs/developer_guides/pipelines/dataparsers.md @@ -117,7 +117,7 @@ dataparser_outputs = dataparser.get_dataparser_outputs(split="train") input_dataset = InputDataset(dataparser_outputs) ``` -You can also pull out information from the DataParserOutputs for other DataMangager components, such as the RayGenerator. The RayGenerator generates RayBundle objects from camera and pixel indices. +You can also pull out information from the DataParserOutputs for other DataManager components, such as the RayGenerator. The RayGenerator generates RayBundle objects from camera and pixel indices. ```python ray_generator = RayGenerator(dataparser_outputs.cameras) diff --git a/docs/developer_guides/viewer/index.md b/docs/developer_guides/viewer/index.md index eebc29d32..93204a203 100644 --- a/docs/developer_guides/viewer/index.md +++ b/docs/developer_guides/viewer/index.md @@ -8,6 +8,7 @@ The viewer is built on [Viser](https://github.com/brentyi/viser/tree/main/viser) :titlesonly: custom_gui +viewer_control local_viewer ``` diff --git a/docs/developer_guides/viewer/viewer_control.md b/docs/developer_guides/viewer/viewer_control.md new file mode 100644 index 000000000..e6a84d3d7 --- /dev/null +++ b/docs/developer_guides/viewer/viewer_control.md @@ -0,0 +1,82 @@ +# Python Viewer Control + +Similar to [`ViewerElements`](./custom_gui.md), Nerfstudio includes supports a Python interface to the viewer through which you can: + +* Set viewer camera pose and FOV +* Set viewer scene crop +* Retrieve the current viewer camera matrix +* Install listeners for click events inside the viewer window + +## Usage + +First, instantiate a `ViewerControl` object as a class variable inside a model file. +Just like `ViewerElements`, you can create an instance inside any class which inherits from `nn.Module` +and is contained within the `Pipeline` object (for example the `Model`) + +```python +from nerfstudio.viewer.server.viewer_elements import ViewerControl + +class MyModel(nn.Module): # Must inherit from nn.Module + def __init__(self): + # Must be a class variable + self.viewer_control = ViewerControl() # no arguments +``` +## Get Camera Matrix +To get the current camera intrinsics and extrinsics, use the `get_camera` function. This returns a `nerfstudio.cameras.cameras.Cameras` object. This object can be used to generate `RayBundles`, retrieve intrinsics and extrinsics, and more. + +```python +from nerfstudio.viewer.server.viewer_elements import ViewerControl, ViewerButton + +class MyModel(nn.Module): # Must inherit from nn.Module + def __init__(self): + ... + def button_cb(button): + # example of using the get_camera function, pass img width and height + # returns a Cameras object with 1 camera + camera = self.viewer_control.get_camera(100,100) + if camera is None: + # returns None when the viewer is not connected yet + return + # get the camera pose + camera_extrinsics_matrix = camera.camera_to_worlds[0,...] # 3x4 matrix + # generate image RayBundle + bundle = camera.generate_rays(camera_indices=0) + # Compute depth, move camera, or whatever you want + ... + self.viewer_button = ViewerButton(name="Dummy Button",cb_hook=button_cb) +``` + +## Set Camera Properties +You can set the viewer camera position and FOV from python. +To set position, you must define a new camera position as well as a 3D "look at" point which the camera aims towards. +```python +from nerfstudio.viewer.server.viewer_elements import ViewerControl,ViewerButton + +class MyModel(nn.Module): # Must inherit from nn.Module + def __init__(self): + ... + def aim_at_origin(button): + # instant=False means the camera smoothly animates + # instant=True means the camera jumps instantly to the pose + self.viewer_control.set_pose(position=(1,1,1),look_at=(0,0,0),instant=False) + self.viewer_button = ViewerButton(name="Dummy Button",cb_hook=button_cb) +``` + +## Double-click Callbacks +We forward *double* clicks inside the viewer to the ViewerControl object, which you can use to interact with the scene. To do this, register a callback using `register_click_cb()`. The click is defined to be a ray that starts at the camera origin and passes through the click point on the screen, in world coordinates. + +```python +from nerfstudio.viewer.server.viewer_elements import ViewerControl,ViewerClick + +class MyModel(nn.Module): # must inherit from nn.Module + def __init__(self): + # Must be a class variable + self.viewer_control = ViewerControl() # no arguments + def click_cb(click: ViewerClick): + print(f"Click at {click.origin} in direction {click.direction}") + self.viewer_control.register_click_cb(click_cb) +``` + +### Thread safety +Just like `ViewerElement` callbacks, click callbacks are asynchronous to training and can potentially interrupt a call to `get_outputs()`. + diff --git a/docs/extensions/sdfstudio.md b/docs/extensions/sdfstudio.md index 4c8f24874..fa2625d0f 100644 --- a/docs/extensions/sdfstudio.md +++ b/docs/extensions/sdfstudio.md @@ -29,7 +29,7 @@ You can learn more about these methods [here](https://github.com/autonomousvisio ## Surface models in nerfstudio -We intend to integrate many of the SDFStudio improvments back into the nerstudio core repository. +We intend to integrate many of the SDFStudio improvements back into the nerfstudio core repository. Supported methods: diff --git a/docs/index.md b/docs/index.md index ee611c573..9ebd68308 100644 --- a/docs/index.md +++ b/docs/index.md @@ -35,7 +35,7 @@ It's as simple as plug and play with nerfstudio! On top of our API, we are committed to providing learning resources to help you understand the basics of (if you're just getting started), and keep up-to-date with (if you're a seasoned veteran) all things NeRF. As researchers, we know just how hard it is to get onboarded with this next-gen technology. So we're here to help with tutorials, documentation, and more! -Finally, have feature requests? Want to add your brand-spankin'-new NeRF model? Have a new dataset? **We welcome any and all [contributions](reference/contributing)!** +Finally, have feature requests? Want to add your brand-spankin'-new NeRF model? Have a new dataset? **We welcome [contributions](reference/contributing)!** Please do not hesitate to reach out to the nerfstudio team with any questions via [Discord](https://discord.gg/uMbNqcraFc). We hope nerfstudio enables you to build faster 🔨 learn together 📚 and contribute to our NeRF community 💖. @@ -99,16 +99,24 @@ This documentation is organized into 3 parts: ## Supported Methods -- [**Nerfacto**](https://github.com/nerfstudio-project/nerfstudio/blob/master/nerfstudio/models/nerfacto.py): our de facto NeRF method combines modules focused on quality with modules focused on faster rendering. Nerfstudio easily lets us experiment with the best of both worlds! -- [NeRF](https://www.matthewtancik.com/nerf): Representing Scenes as Neural Radiance Fields for View Synthesis -- [Instant NGP](https://nvlabs.github.io/instant-ngp/): Instant Neural Graphics Primitives with a Multiresolution Hash Encoding -- [Mipnerf](https://jonbarron.info/mipnerf/): A Multiscale Representation for Anti-Aliasing Neural Radiance Fields -- [NerfW](https://nerf-w.github.io/): Neural Radiance Fields for Unconstrained Photo Collections -- [Semantic NeRF](https://shuaifengzhi.com/Semantic-NeRF/): In-Place Scene Labelling and Understanding with Implicit Scene Representation +### Included Methods -We'll be constantly growing this list! So make sure to check back in to see our updates. +- [**Nerfacto**](nerfology/methods/nerfacto.md): Recommended method, integrates mutiple methods into one. +- [Instant-NGP](nerfology/methods/instant_ngp.md): Instant Neural Graphics Primitives with a Multiresolution Hash Encoding +- [NeRF](nerfology/methods/nerf.md): OG Neural Radiance Fields +- [Mip-NeRF](nerfology/methods/mipnerf.md): A Multiscale Representation for Anti-Aliasing Neural Radiance Fields +- [TensoRF](nerfology/methods/tensorf.md): Tensorial Radiance Fields -**Eager to contribute?** We'd love to see you use nerfstudio in implementing new (or even existing) methods! Feel free to contact us directly or view our [Contributor's Guide](reference/contributing) to see how you can get your model on this list! +(third_party_methods)= + +### Third-party Methods + +- [Instruct-NeRF2NeRF](nerfology/methods/in2n.md): Editing 3D Scenes with Instructions +- [K-Planes](nerfology/methods/kplanes.md): Unified 3D and 4D Radiance Fields +- [LERF](nerfology/methods/lerf.md): Language Embedded Radiance Fields +- [Tetra-NeRF](nerfology/methods/tetranerf.md): Representing Neural Radiance Fields Using Tetrahedra + +**Eager to contribute a method?** We'd love to see you use nerfstudio in implementing new (or even existing) methods! Please view our {ref}`guide` for more details about how to add to this list! ## Quicklinks @@ -168,13 +176,17 @@ You can find a paper writeup of the framework on [arXiv](https://arxiv.org/abs/2 If you use this library or find the documentation useful for your research, please consider citing: ```none -@article{nerfstudio, - author = {Tancik, Matthew and Weber, Ethan and Ng, Evonne and Li, Ruilong and Yi, - Brent and Kerr, Justin and Wang, Terrance and Kristoffersen, Alexander and Austin, - Jake and Salahi, Kamyar and Ahuja, Abhik and McAllister, David and Kanazawa, Angjoo}, - title = {Nerfstudio: A Modular Framework for Neural Radiance Field Development}, - journal = {arXiv preprint arXiv:2302.04264}, - year = {2023}, +@inproceedings{nerfstudio, + title = {Nerfstudio: A Modular Framework for Neural Radiance Field Development}, + author = { + Tancik, Matthew and Weber, Ethan and Ng, Evonne and Li, Ruilong and Yi, Brent + and Kerr, Justin and Wang, Terrance and Kristoffersen, Alexander and Austin, + Jake and Salahi, Kamyar and Ahuja, Abhik and McAllister, David and Kanazawa, + Angjoo + }, + year = 2023, + booktitle = {ACM SIGGRAPH 2023 Conference Proceedings}, + series = {SIGGRAPH '23} } ``` @@ -183,3 +195,13 @@ If you use this library or find the documentation useful for your research, plea + +## Maintainers + +| | Nerfstudio Discord | Affiliation | +| -------------------------------------------- | ------------------ | ------------------------------------ | +| [Justin Kerr](https://kerrj.github.io/) | justin.kerr | UC Berkeley | +| [Jonáš Kulhánek](https://jkulhanek.com/) | jkulhanek | Czech Technical University in Prague | +| [Matt Tancik](https://www.matthewtancik.com) | tancik | UC Berkeley | +| [Ethan Weber](https://ethanweber.me/) | ethanweber | UC Berkeley | +| [Brent Yi](https://github.com/brentyi) | brent | UC Berkeley | diff --git a/docs/nerfology/methods/imgs/ingp/hash_figure.png b/docs/nerfology/methods/imgs/ingp/hash_figure.png new file mode 100644 index 000000000..602d5fdbd Binary files /dev/null and b/docs/nerfology/methods/imgs/ingp/hash_figure.png differ diff --git a/docs/nerfology/methods/in2n.md b/docs/nerfology/methods/in2n.md index bd9ebccb6..3b75d9a51 100644 --- a/docs/nerfology/methods/in2n.md +++ b/docs/nerfology/methods/in2n.md @@ -83,7 +83,7 @@ The code snippet for how an image is edited in the pipeline can be found [here]( When NeRF training starts, the dataset consists of the original, unedited images used to train the original scene. These images are saved separately to use as conditioning for InstructPix2Pix. At each optimization step, some number of NeRF optimization steps are performed, and then some number of images (often just one) are updated. The images are randomly ordered prior to training and then at each step, the images are chosen in order to edit. Once an image has been edited, it is replaced in the dataset. Importantly, at each NeRF step, rays are sampled across the entire dataset, meaning there is a mixed source of supervision between edited images and unedited images. This allows for a gradual optimization that balances maintaining the 3D structure and consistency of the NeRF as well as performing the edit. -At early iterations of this process, the edited images may be inconistent with one another, as InstructPix2Pix often doesn't perform consistent edits across viewpoints. However, over time, since images are edited using the current render of the NeRF, the edits begin to converge towards a globally consistent depiction of the underlying scene. Here is an example of how the underlying dataset evolves and becomes more consistent. +At early iterations of this process, the edited images may be inconsistent with one another, as InstructPix2Pix often doesn't perform consistent edits across viewpoints. However, over time, since images are edited using the current render of the NeRF, the edits begin to converge towards a globally consistent depiction of the underlying scene. Here is an example of how the underlying dataset evolves and becomes more consistent.