diff --git a/docs/conf.py b/docs/conf.py
index b3bf31ea5..d4ebac9f8 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -15,8 +15,7 @@ import os
import sys
sys.path.insert(0, os.path.abspath(".."))
-import nerfstudio.configs.base
-import nerfstudio.datamanagers.base
+import nerfstudio.configs.base_config
# -- Project information -----------------------------------------------------
diff --git a/docs/guides/model_components/visualize_encoders.ipynb b/docs/guides/model_components/visualize_encoders.ipynb
index 3dd590422..86001abd9 100644
--- a/docs/guides/model_components/visualize_encoders.ipynb
+++ b/docs/guides/model_components/visualize_encoders.ipynb
@@ -1,561 +1,561 @@
{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Field Encoders"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "(nerf)=\n",
- "## NeRF Positional Encoding\n",
- "First introduced in the original NeRF paper. This encoding assumes the inputs are between zero and one and can opperate on any dimensional input."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {
- "tags": [
- "hide-input"
- ]
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Input Values:\n"
- ]
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Field Encoders"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "(nerf)=\n",
+ "## NeRF Positional Encoding\n",
+ "First introduced in the original NeRF paper. This encoding assumes the inputs are between zero and one and can opperate on any dimensional input."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "tags": [
+ "hide-input"
+ ]
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Input Values:\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "![](\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIAAAACACAIAAABMXPacAAABuUlEQVR4nO3aQa7gNgxEQdvqGcwm586lk+85Ri386gQEiCYl2fc/f/694uz3++gaPm2/36Nr+LTtKgHSftUAantvXcOnNYKwGoDt1ACqHYCVAGy7SoC0UwOorbcgqgRgNQDb6RhKdQrCGkFYIwgrAVgNwDZdwce1A7D1T4TVDsBKAFYDsJ0mEFUCsBqANYKwGoDt3K+u4dNKALbzlACpBmA7/RZE7SkBVCMI23l+dA2ftnNKgNQOwBpBWEsY23NKgFQCsD3tAKoljO3pHkA1grBGEFYCsO4BWAnA2gFYr6FYOwBrB2A1AKsBWDsAKwFYx1Bsz/7XNXxaIwirAVgNwPokie0uAVQjCKsB2O52AFUCsBqA7e41lOoxDusegPVBBtvdCKJKANYOwGoA1nM0VgKwbsJYN2GsBGDtAKwEYH0RwxpBWCMIqwFYz9FYCcBqALarYyjVMRRrBGG7agDVUwRWArB2ANYxFGsEYSUAawdgJQBrB2AlANtbA6gSgO1aDZAaQdiuo0v4thKA7Zou4dv2NoKovefWNXza3kYQ1QjC9q4RJLUDsP00gqiWMLa3DzLU3v2na/i0v30YbAheKpYKAAAAAElFTkSuQmCC\"/) | ![](\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIAAAACACAIAAABMXPacAAAB00lEQVR4nO3RMY7dQAwEUZI9XjjxuX1pe+hAK/jfoJJ6gdDgKKv+9fP3187X5tT8qDnbp+bUpOZsp+ZUp/rspDrV2T7veC6nKtupSj3fStf/0ZXedGU2s5ma73GTnWfMTm5mZ27mTt4xd3Lfvc+ej/vMzvn7cdx+d2ffcWe2505uz05uz+335+/X55jb77Fz+7nn1nus3M7We6zn6WNvtrJ1nlGbrVOb2vQ+4/Smb2pPbe6eP1NCGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmANg/vMBrChrbL84AAAAASUVORK5CYII=\"/) |
"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Encoded Values:\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ ""
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Encoded Integrate Values:\n",
+ "Covariance Magnitude: 0.01\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ ""
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Covariance Magnitude: 0.1\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ ""
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Covariance Magnitude: 1\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ ""
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "# COLLAPSED\n",
+ "import torch\n",
+ "import mediapy as media\n",
+ "from nerfstudio.field_components import encoding\n",
+ "\n",
+ "num_frequencies = 4\n",
+ "min_freq_exp = 0\n",
+ "max_freq_exp = 6\n",
+ "include_input = False\n",
+ "resolution = 128\n",
+ "covariance_magnitudes = [0.01, 0.1, 1]\n",
+ "\n",
+ "encoder = encoding.NeRFEncoding(\n",
+ " in_dim=2,\n",
+ " num_frequencies=num_frequencies,\n",
+ " min_freq_exp=min_freq_exp,\n",
+ " max_freq_exp=max_freq_exp,\n",
+ " include_input=include_input,\n",
+ ")\n",
+ "\n",
+ "x_samples = torch.linspace(0, 1, resolution)\n",
+ "grid = torch.stack(torch.meshgrid([x_samples, x_samples], indexing=\"ij\"), dim=-1)\n",
+ "\n",
+ "encoded_values = encoder(grid)\n",
+ "\n",
+ "print(\"Input Values:\")\n",
+ "media.show_images(torch.moveaxis(grid, 2, 0), cmap=\"plasma\", border=True)\n",
+ "print(\"Encoded Values:\")\n",
+ "media.show_images(torch.moveaxis(encoded_values, 2, 0), vmin=-1, vmax=1, cmap=\"plasma\", border=True)\n",
+ "\n",
+ "print(\"Encoded Integrate Values:\")\n",
+ "for covariance_magnitude in covariance_magnitudes:\n",
+ " print(f\"Covariance Magnitude: {covariance_magnitude}\")\n",
+ " covs = torch.eye(2)[None, None, :, :] * covariance_magnitude\n",
+ " encoded_values = encoder(grid, covs=covs)\n",
+ " media.show_images(torch.moveaxis(encoded_values, 2, 0), vmin=-1, vmax=1, cmap=\"plasma\", border=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "(rff)=\n",
+ "## Random Fourier Feature (RFF) Encoding\n",
+ "This encoding assumes the inputs are between zero and one and can opperate on any dimensional input."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {
+ "tags": [
+ "hide-input"
+ ]
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Input Values:\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ " | |
"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Encoded Values:\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ ""
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Encoded Integrate Values:\n",
+ "Covariance Magnitude: 0.001\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ ""
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Covariance Magnitude: 0.01\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ ""
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Covariance Magnitude: 0.1\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ ""
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "# COLLAPSED\n",
+ "import torch\n",
+ "import mediapy as media\n",
+ "from nerfstudio.field_components import encoding\n",
+ "\n",
+ "num_frequencies = 8\n",
+ "scale = 10\n",
+ "resolution = 128\n",
+ "covariance_magnitudes = [0.001, 0.01, 0.1]\n",
+ "\n",
+ "encoder = encoding.RFFEncoding(in_dim=2, num_frequencies=num_frequencies, scale=scale)\n",
+ "\n",
+ "x_samples = torch.linspace(0, 1, resolution)\n",
+ "grid = torch.stack(torch.meshgrid([x_samples, x_samples], indexing=\"ij\"), dim=-1)\n",
+ "\n",
+ "encoded_values = encoder(grid)\n",
+ "\n",
+ "print(\"Input Values:\")\n",
+ "media.show_images(torch.moveaxis(grid, 2, 0), cmap=\"plasma\", border=True)\n",
+ "print(\"Encoded Values:\")\n",
+ "media.show_images(torch.moveaxis(encoded_values, 2, 0), cmap=\"plasma\", vmin=-1, vmax=1, border=True)\n",
+ "\n",
+ "print(\"Encoded Integrate Values:\")\n",
+ "for covariance_magnitude in covariance_magnitudes:\n",
+ " print(f\"Covariance Magnitude: {covariance_magnitude}\")\n",
+ " covs = torch.eye(2)[None, None, :, :] * covariance_magnitude\n",
+ " encoded_values = encoder(grid, covs=covs)\n",
+ " media.show_images(torch.moveaxis(encoded_values, 2, 0), cmap=\"plasma\", vmin=-1, vmax=1, border=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "(hash)=\n",
+ "## Hash Encoding\n",
+ "The hash incoding was originally introduced in Instant-NGP. The encoding is optimized during training. This is a visualization of the initialization."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "tags": [
+ "hide-input"
+ ]
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Input Values:\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ ""
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Encoded Values:\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ ""
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "# COLLAPSED\n",
+ "import torch\n",
+ "import mediapy as media\n",
+ "from nerfstudio.field_components import encoding\n",
+ "\n",
+ "num_levels = 8\n",
+ "min_res = 2\n",
+ "max_res = 128\n",
+ "hash_table_size = 2**4 # Typically much larger tables are used\n",
+ "\n",
+ "resolution = 128\n",
+ "slice = 0\n",
+ "\n",
+ "# Fixing features_per_level to 3 for easy RGB visualization. Typical value is 2 in networks\n",
+ "features_per_level = 3\n",
+ "\n",
+ "encoder = encoding.HashEncoding(\n",
+ " num_levels=num_levels,\n",
+ " min_res=min_res,\n",
+ " max_res=max_res,\n",
+ " hash_table_size=hash_table_size,\n",
+ " features_per_level=features_per_level,\n",
+ " hash_init_scale=0.001,\n",
+ ")\n",
+ "\n",
+ "x_samples = torch.linspace(0, 1, resolution)\n",
+ "grid = torch.stack(torch.meshgrid([x_samples, x_samples, x_samples], indexing=\"ij\"), dim=-1)\n",
+ "\n",
+ "encoded_values = encoder(grid)\n",
+ "\n",
+ "grid_slice = grid[slice, ...]\n",
+ "encoded_values_slice = encoded_values[slice, ...]\n",
+ "\n",
+ "print(\"Input Values:\")\n",
+ "media.show_images(torch.moveaxis(grid_slice, 2, 0), cmap=\"plasma\", border=True)\n",
+ "\n",
+ "print(\"Encoded Values:\")\n",
+ "encoded_images = encoded_values_slice.view(resolution, resolution, num_levels, 3)\n",
+ "encoded_images = torch.moveaxis(encoded_images, 2, 0)\n",
+ "encoded_images -= torch.min(encoded_images)\n",
+ "encoded_images /= torch.max(encoded_images)\n",
+ "media.show_images(encoded_images.detach().numpy(), cmap=\"plasma\", border=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "(spherical)=\n",
+ "## Spherical Harmonic Encoding\n",
+ "Encode direction using spherical harmoincs. (Mostly used to encode viewing direction)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "tags": [
+ "hide-input"
+ ]
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Level: 1\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "![](\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAJYAAABkCAIAAADrOV6nAAABA0lEQVR4nO3RsQkAIQDAwPcrG+d2bYewkMDdBIGMNfdH2f86gFsW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYdwDkygFkzaTKPwAAAABJRU5ErkJggg==\"/) |
"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Level: 2\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ ""
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Level: 3\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ ""
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Level: 4\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ ""
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "# COLLAPSED\n",
+ "import torch\n",
+ "import mediapy as media\n",
+ "from nerfstudio.field_components import encoding\n",
+ "\n",
+ "levels = 4\n",
+ "\n",
+ "height = 100\n",
+ "width = 150\n",
+ "\n",
+ "encoder = encoding.SHEncoding(levels=levels)\n",
+ "\n",
+ "theta = torch.linspace(-torch.pi, torch.pi, width)\n",
+ "phi = torch.linspace(0, torch.pi, height)\n",
+ "[theta, phi] = torch.meshgrid([theta, phi], indexing=\"xy\")\n",
+ "\n",
+ "directions = torch.stack([torch.cos(theta) * torch.sin(phi), torch.sin(theta) * torch.sin(phi), torch.cos(phi)], dim=-1)\n",
+ "\n",
+ "encoded_values = encoder(directions)\n",
+ "encoded_values = torch.moveaxis(encoded_values, 2, 0)\n",
+ "\n",
+ "for level in range(levels):\n",
+ " print(f\"Level: {level+1}\")\n",
+ " media.show_images(encoded_values[level**2 : (level + 1) ** 2, ...], cmap=\"plasma\", border=True)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3.8.12 ('nerfactory')",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.12"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "34c28001ff35fb390494047002768a8182dcf55b1b11415165e62ea61557ab83"
+ }
+ }
},
- {
- "data": {
- "text/html": [
- " | |
"
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Encoded Values:\n"
- ]
- },
- {
- "data": {
- "text/html": [
- ""
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Encoded Integrate Values:\n",
- "Covariance Magnitude: 0.01\n"
- ]
- },
- {
- "data": {
- "text/html": [
- ""
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Covariance Magnitude: 0.1\n"
- ]
- },
- {
- "data": {
- "text/html": [
- ""
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Covariance Magnitude: 1\n"
- ]
- },
- {
- "data": {
- "text/html": [
- ""
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
- "source": [
- "# COLLAPSED\n",
- "import torch\n",
- "import mediapy as media\n",
- "from nerfstudio.fields.modules import encoding\n",
- "\n",
- "num_frequencies = 4\n",
- "min_freq_exp = 0\n",
- "max_freq_exp = 6\n",
- "include_input = False\n",
- "resolution = 128\n",
- "covariance_magnitudes = [0.01, 0.1, 1]\n",
- "\n",
- "encoder = encoding.NeRFEncoding(\n",
- " in_dim=2,\n",
- " num_frequencies=num_frequencies,\n",
- " min_freq_exp=min_freq_exp,\n",
- " max_freq_exp=max_freq_exp,\n",
- " include_input=include_input,\n",
- ")\n",
- "\n",
- "x_samples = torch.linspace(0, 1, resolution)\n",
- "grid = torch.stack(torch.meshgrid([x_samples, x_samples], indexing=\"ij\"), dim=-1)\n",
- "\n",
- "encoded_values = encoder(grid)\n",
- "\n",
- "print(\"Input Values:\")\n",
- "media.show_images(torch.moveaxis(grid, 2, 0), cmap=\"plasma\", border=True)\n",
- "print(\"Encoded Values:\")\n",
- "media.show_images(torch.moveaxis(encoded_values, 2, 0), vmin=-1, vmax=1, cmap=\"plasma\", border=True)\n",
- "\n",
- "print(\"Encoded Integrate Values:\")\n",
- "for covariance_magnitude in covariance_magnitudes:\n",
- " print(f\"Covariance Magnitude: {covariance_magnitude}\")\n",
- " covs = torch.eye(2)[None, None, :, :] * covariance_magnitude\n",
- " encoded_values = encoder(grid, covs=covs)\n",
- " media.show_images(torch.moveaxis(encoded_values, 2, 0), vmin=-1, vmax=1, cmap=\"plasma\", border=True)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "(rff)=\n",
- "## Random Fourier Feature (RFF) Encoding\n",
- "This encoding assumes the inputs are between zero and one and can opperate on any dimensional input."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 25,
- "metadata": {
- "tags": [
- "hide-input"
- ]
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Input Values:\n"
- ]
- },
- {
- "data": {
- "text/html": [
- " | |
"
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Encoded Values:\n"
- ]
- },
- {
- "data": {
- "text/html": [
- ""
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Encoded Integrate Values:\n",
- "Covariance Magnitude: 0.001\n"
- ]
- },
- {
- "data": {
- "text/html": [
- ""
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Covariance Magnitude: 0.01\n"
- ]
- },
- {
- "data": {
- "text/html": [
- ""
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Covariance Magnitude: 0.1\n"
- ]
- },
- {
- "data": {
- "text/html": [
- ""
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
- "source": [
- "# COLLAPSED\n",
- "import torch\n",
- "import mediapy as media\n",
- "from nerfstudio.fields.modules import encoding\n",
- "\n",
- "num_frequencies = 8\n",
- "scale = 10\n",
- "resolution = 128\n",
- "covariance_magnitudes = [0.001, 0.01, 0.1]\n",
- "\n",
- "encoder = encoding.RFFEncoding(in_dim=2, num_frequencies=num_frequencies, scale=scale)\n",
- "\n",
- "x_samples = torch.linspace(0, 1, resolution)\n",
- "grid = torch.stack(torch.meshgrid([x_samples, x_samples], indexing=\"ij\"), dim=-1)\n",
- "\n",
- "encoded_values = encoder(grid)\n",
- "\n",
- "print(\"Input Values:\")\n",
- "media.show_images(torch.moveaxis(grid, 2, 0), cmap=\"plasma\", border=True)\n",
- "print(\"Encoded Values:\")\n",
- "media.show_images(torch.moveaxis(encoded_values, 2, 0), cmap=\"plasma\", vmin=-1, vmax=1, border=True)\n",
- "\n",
- "print(\"Encoded Integrate Values:\")\n",
- "for covariance_magnitude in covariance_magnitudes:\n",
- " print(f\"Covariance Magnitude: {covariance_magnitude}\")\n",
- " covs = torch.eye(2)[None, None, :, :] * covariance_magnitude\n",
- " encoded_values = encoder(grid, covs=covs)\n",
- " media.show_images(torch.moveaxis(encoded_values, 2, 0), cmap=\"plasma\", vmin=-1, vmax=1, border=True)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "(hash)=\n",
- "## Hash Encoding\n",
- "The hash incoding was originally introduced in Instant-NGP. The encoding is optimized during training. This is a visualization of the initialization."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {
- "tags": [
- "hide-input"
- ]
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Input Values:\n"
- ]
- },
- {
- "data": {
- "text/html": [
- ""
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Encoded Values:\n"
- ]
- },
- {
- "data": {
- "text/html": [
- ""
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
- "source": [
- "# COLLAPSED\n",
- "import torch\n",
- "import mediapy as media\n",
- "from nerfstudio.fields.modules import encoding\n",
- "\n",
- "num_levels = 8\n",
- "min_res = 2\n",
- "max_res = 128\n",
- "hash_table_size = 2**4 # Typically much larger tables are used\n",
- "\n",
- "resolution = 128\n",
- "slice = 0\n",
- "\n",
- "# Fixing features_per_level to 3 for easy RGB visualization. Typical value is 2 in networks\n",
- "features_per_level = 3\n",
- "\n",
- "encoder = encoding.HashEncoding(\n",
- " num_levels=num_levels,\n",
- " min_res=min_res,\n",
- " max_res=max_res,\n",
- " hash_table_size=hash_table_size,\n",
- " features_per_level=features_per_level,\n",
- " hash_init_scale=0.001,\n",
- ")\n",
- "\n",
- "x_samples = torch.linspace(0, 1, resolution)\n",
- "grid = torch.stack(torch.meshgrid([x_samples, x_samples, x_samples], indexing=\"ij\"), dim=-1)\n",
- "\n",
- "encoded_values = encoder(grid)\n",
- "\n",
- "grid_slice = grid[slice, ...]\n",
- "encoded_values_slice = encoded_values[slice, ...]\n",
- "\n",
- "print(\"Input Values:\")\n",
- "media.show_images(torch.moveaxis(grid_slice, 2, 0), cmap=\"plasma\", border=True)\n",
- "\n",
- "print(\"Encoded Values:\")\n",
- "encoded_images = encoded_values_slice.view(resolution, resolution, num_levels, 3)\n",
- "encoded_images = torch.moveaxis(encoded_images, 2, 0)\n",
- "encoded_images -= torch.min(encoded_images)\n",
- "encoded_images /= torch.max(encoded_images)\n",
- "media.show_images(encoded_images.detach().numpy(), cmap=\"plasma\", border=True)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "(spherical)=\n",
- "## Spherical Harmonic Encoding\n",
- "Encode direction using spherical harmoincs. (Mostly used to encode viewing direction)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {
- "tags": [
- "hide-input"
- ]
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Level: 1\n"
- ]
- },
- {
- "data": {
- "text/html": [
- " |
"
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Level: 2\n"
- ]
- },
- {
- "data": {
- "text/html": [
- ""
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Level: 3\n"
- ]
- },
- {
- "data": {
- "text/html": [
- ""
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Level: 4\n"
- ]
- },
- {
- "data": {
- "text/html": [
- ""
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
- "source": [
- "# COLLAPSED\n",
- "import torch\n",
- "import mediapy as media\n",
- "from nerfstudio.fields.modules import encoding\n",
- "\n",
- "levels = 4\n",
- "\n",
- "height = 100\n",
- "width = 150\n",
- "\n",
- "encoder = encoding.SHEncoding(levels=levels)\n",
- "\n",
- "theta = torch.linspace(-torch.pi, torch.pi, width)\n",
- "phi = torch.linspace(0, torch.pi, height)\n",
- "[theta, phi] = torch.meshgrid([theta, phi], indexing=\"xy\")\n",
- "\n",
- "directions = torch.stack([torch.cos(theta) * torch.sin(phi), torch.sin(theta) * torch.sin(phi), torch.cos(phi)], dim=-1)\n",
- "\n",
- "encoded_values = encoder(directions)\n",
- "encoded_values = torch.moveaxis(encoded_values, 2, 0)\n",
- "\n",
- "for level in range(levels):\n",
- " print(f\"Level: {level+1}\")\n",
- " media.show_images(encoded_values[level**2 : (level + 1) ** 2, ...], cmap=\"plasma\", border=True)"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3.8.12 ('nerfactory')",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.8.12"
- },
- "vscode": {
- "interpreter": {
- "hash": "34c28001ff35fb390494047002768a8182dcf55b1b11415165e62ea61557ab83"
- }
- }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
\ No newline at end of file
diff --git a/docs/guides/model_components/visualize_samplers.ipynb b/docs/guides/model_components/visualize_samplers.ipynb
index 1c487eee1..fc45475b7 100644
--- a/docs/guides/model_components/visualize_samplers.ipynb
+++ b/docs/guides/model_components/visualize_samplers.ipynb
@@ -8,7 +8,7 @@
"\n",
"## Overview\n",
"\n",
- "Once we have a set of cameras, we want to cast camera rays associated with each pixel. Along these ray we will sample the _field_ and aggregate the samples to predict the pixels value (ie. color). The parameterization of the samples are described [here](./visualize_samples.ipynb) however we must decide where to place these samples along a ray. For this task we will use a [`Sampler`](../../reference/api/graphs/modules/ray_sampler.rst).\n",
+ "Once we have a set of cameras, we want to cast camera rays associated with each pixel. Along these ray we will sample the _field_ and aggregate the samples to predict the pixels value (ie. color). The parameterization of the samples are described [here](./visualize_samples.ipynb) however we must decide where to place these samples along a ray. For this task we will use a [`Sampler`](../../reference/api/model_components/ray_sampler.rst).\n",
"\n",
"In the ideal world we would compute many dense samples along a ray. Unfortunatly each additional sample adds a computation cost to the system as it needs to be processed by the _field_ which is often a neural network. As a result it is common for NeRF methods to use on the order of 100 samples. Therefore we want to optimize where those samples are placed in the scene.\n",
"\n",
@@ -5115,7 +5115,7 @@
"# COLLAPSED\n",
"import torch\n",
"from nerfstudio.cameras.rays import RayBundle\n",
- "from nerfstudio.models.modules import ray_sampler\n",
+ "from nerfstudio.model_components import ray_sampler\n",
"import plotly.graph_objects as go\n",
"from plotly.subplots import make_subplots\n",
"\n",
@@ -7223,7 +7223,7 @@
"# COLLAPSED\n",
"import torch\n",
"from nerfstudio.cameras.rays import RayBundle\n",
- "from nerfstudio.models.modules import ray_sampler\n",
+ "from nerfstudio.model_components import ray_sampler\n",
"import plotly.graph_objects as go\n",
"from plotly.subplots import make_subplots\n",
"\n",
diff --git a/docs/guides/model_components/visualize_spatial_distortions.ipynb b/docs/guides/model_components/visualize_spatial_distortions.ipynb
index ff06cca14..fd8c5f377 100644
--- a/docs/guides/model_components/visualize_spatial_distortions.ipynb
+++ b/docs/guides/model_components/visualize_spatial_distortions.ipynb
@@ -1,234427 +1,234427 @@
{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {
- "tags": []
- },
- "source": [
- "# Spatial Distortions\n",
- "\n",
- "If you are trying to reconstruct an object floating in an empty void, you can stop reading. However if you are trying to reconstruct a scene or object from images, you may wish to consider adding a spatial distortion.\n",
- "\n",
- "When rendering a target view of a scene, the camera will emit a camera ray for each pixel and query the scene at points along this ray. We can choose where to query these points using different [samplers](visualize_samplers.ipynb). These samplers have some notion of _bounds_ that define where the ray should start and terminate. If you know that everything in your scenes exists within some predefined bounds (ie. a cube that a room fits in) then the sampler will properly sample the entire space. If however the scene is unbounded (ie. an outdoor scene) defining where to stop sampling is challenging. One option to increase the far sampling distance to a large value (ie. 1km). Alternatively we can warp the space into a fixed volume. Below are supported distortions.\n"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "tags": []
- },
- "source": [
- "## Scene Contraction\n",
- "\n",
- "Contract unbounded space into a ball of radius 2. This contraction was proposed in [MipNeRF-360](https://jonbarron.info/mipnerf360/). Samples within the unit ball are not modified, whereas sample outside the unit ball are contracted to fit within the ball of radius 2.\n",
- "\n",
- "We use the following contraction equation:\n",
- "\n",
- "$$\n",
- " f(x) = \\begin{cases}\n",
- " x & ||x|| \\leq 1 \\\\\n",
- " (2 - \\frac{1}{||x||})(\\frac{x}{||x||}) & ||x|| > 1\n",
- " \\end{cases}\n",
- "$$\n",
- "\n",
- "Below we visualize a ray before and after scene contraction. Visualized are 95% confidence intervals for the multivariate Gaussians for each sample location ([this guide](visualize_samples.ipynb) explains why the samples are represented by Gaussians).\n",
- "We are also visualizing both a unit sphere and a radius 2 sphere.\n",
- "\n",
- "(before)=\n",
- "### Before Contraction\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {
- "tags": [
- "remove-cell"
- ]
- },
- "outputs": [],
- "source": [
- "# HIDDEN\n",
- "import torch\n",
- "import plotly.graph_objects as go\n",
- "\n",
- "torch.manual_seed(4)\n",
- "\n",
- "webdocs_layout = go.Layout(\n",
- " scene=dict(\n",
- " aspectmode=\"data\",\n",
- " xaxis=dict(showspikes=False),\n",
- " yaxis=dict(showspikes=False),\n",
- " zaxis=dict(showspikes=False),\n",
- " ),\n",
- " scene_camera=dict(up=dict(x=0, y=1, z=0)),\n",
- " margin=dict(r=0, b=10, l=0, t=10),\n",
- " hovermode=False,\n",
- " showlegend=False,\n",
- " paper_bgcolor=\"rgba(0,0,0,0)\",\n",
- ")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {
- "tags": [
- "hide-input"
- ]
- },
- "outputs": [
- {
- "data": {
- "text/html": [
- " \n",
- " "
- ]
- },
- "metadata": {},
- "output_type": "display_data"
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "tags": []
+ },
+ "source": [
+ "# Spatial Distortions\n",
+ "\n",
+ "If you are trying to reconstruct an object floating in an empty void, you can stop reading. However if you are trying to reconstruct a scene or object from images, you may wish to consider adding a spatial distortion.\n",
+ "\n",
+ "When rendering a target view of a scene, the camera will emit a camera ray for each pixel and query the scene at points along this ray. We can choose where to query these points using different [samplers](visualize_samplers.ipynb). These samplers have some notion of _bounds_ that define where the ray should start and terminate. If you know that everything in your scenes exists within some predefined bounds (ie. a cube that a room fits in) then the sampler will properly sample the entire space. If however the scene is unbounded (ie. an outdoor scene) defining where to stop sampling is challenging. One option to increase the far sampling distance to a large value (ie. 1km). Alternatively we can warp the space into a fixed volume. Below are supported distortions.\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "tags": []
+ },
+ "source": [
+ "## Scene Contraction\n",
+ "\n",
+ "Contract unbounded space into a ball of radius 2. This contraction was proposed in [MipNeRF-360](https://jonbarron.info/mipnerf360/). Samples within the unit ball are not modified, whereas sample outside the unit ball are contracted to fit within the ball of radius 2.\n",
+ "\n",
+ "We use the following contraction equation:\n",
+ "\n",
+ "$$\n",
+ " f(x) = \\begin{cases}\n",
+ " x & ||x|| \\leq 1 \\\\\n",
+ " (2 - \\frac{1}{||x||})(\\frac{x}{||x||}) & ||x|| > 1\n",
+ " \\end{cases}\n",
+ "$$\n",
+ "\n",
+ "Below we visualize a ray before and after scene contraction. Visualized are 95% confidence intervals for the multivariate Gaussians for each sample location ([this guide](visualize_samples.ipynb) explains why the samples are represented by Gaussians).\n",
+ "We are also visualizing both a unit sphere and a radius 2 sphere.\n",
+ "\n",
+ "(before)=\n",
+ "### Before Contraction\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "tags": [
+ "remove-cell"
+ ]
+ },
+ "outputs": [],
+ "source": [
+ "# HIDDEN\n",
+ "import torch\n",
+ "import plotly.graph_objects as go\n",
+ "\n",
+ "torch.manual_seed(4)\n",
+ "\n",
+ "webdocs_layout = go.Layout(\n",
+ " scene=dict(\n",
+ " aspectmode=\"data\",\n",
+ " xaxis=dict(showspikes=False),\n",
+ " yaxis=dict(showspikes=False),\n",
+ " zaxis=dict(showspikes=False),\n",
+ " ),\n",
+ " scene_camera=dict(up=dict(x=0, y=1, z=0)),\n",
+ " margin=dict(r=0, b=10, l=0, t=10),\n",
+ " hovermode=False,\n",
+ " showlegend=False,\n",
+ " paper_bgcolor=\"rgba(0,0,0,0)\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "tags": [
+ "hide-input"
+ ]
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "