{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Field Encoders"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(nerf)=\n",
"## NeRF Positional Encoding\n",
"First introduced in the original NeRF paper. This encoding assumes the inputs are between zero and one and can opperate on any dimensional input."
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"tags": [
"hide-input"
]
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Input Values:\n"
]
},
{
"data": {
"text/html": [
"![](\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIAAAACACAIAAABMXPacAAABuUlEQVR4nO3aQa7gNgxEQdvqGcwm586lk+85Ri386gQEiCYl2fc/f/694uz3++gaPm2/36Nr+LTtKgHSftUAantvXcOnNYKwGoDt1ACqHYCVAGy7SoC0UwOorbcgqgRgNQDb6RhKdQrCGkFYIwgrAVgNwDZdwce1A7D1T4TVDsBKAFYDsJ0mEFUCsBqANYKwGoDt3K+u4dNKALbzlACpBmA7/RZE7SkBVCMI23l+dA2ftnNKgNQOwBpBWEsY23NKgFQCsD3tAKoljO3pHkA1grBGEFYCsO4BWAnA2gFYr6FYOwBrB2A1AKsBWDsAKwFYx1Bsz/7XNXxaIwirAVgNwPokie0uAVQjCKsB2O52AFUCsBqA7e41lOoxDusegPVBBtvdCKJKANYOwGoA1nM0VgKwbsJYN2GsBGDtAKwEYH0RwxpBWCMIqwFYz9FYCcBqALarYyjVMRRrBGG7agDVUwRWArB2ANYxFGsEYSUAawdgJQBrB2AlANtbA6gSgO1aDZAaQdiuo0v4thKA7Zou4dv2NoKovefWNXza3kYQ1QjC9q4RJLUDsP00gqiWMLa3DzLU3v2na/i0v30YbAheKpYKAAAAAElFTkSuQmCC\"/) | ![](\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIAAAACACAIAAABMXPacAAAB00lEQVR4nO3RMY7dQAwEUZI9XjjxuX1pe+hAK/jfoJJ6gdDgKKv+9fP3187X5tT8qDnbp+bUpOZsp+ZUp/rspDrV2T7veC6nKtupSj3fStf/0ZXedGU2s5ma73GTnWfMTm5mZ27mTt4xd3Lfvc+ej/vMzvn7cdx+d2ffcWe2505uz05uz+335+/X55jb77Fz+7nn1nus3M7We6zn6WNvtrJ1nlGbrVOb2vQ+4/Smb2pPbe6eP1NCGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmAJgBYAaAGQBmANg/vMBrChrbL84AAAAASUVORK5CYII=\"/) |
"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Encoded Values:\n"
]
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Encoded Integrate Values:\n",
"Covariance Magnitude: 0.01\n"
]
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Covariance Magnitude: 0.1\n"
]
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Covariance Magnitude: 1\n"
]
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# COLLAPSED\n",
"import torch\n",
"import mediapy as media\n",
"from nerfstudio.field_components import encoding\n",
"\n",
"num_frequencies = 4\n",
"min_freq_exp = 0\n",
"max_freq_exp = 6\n",
"include_input = False\n",
"resolution = 128\n",
"covariance_magnitudes = [0.01, 0.1, 1]\n",
"\n",
"encoder = encoding.NeRFEncoding(\n",
" in_dim=2,\n",
" num_frequencies=num_frequencies,\n",
" min_freq_exp=min_freq_exp,\n",
" max_freq_exp=max_freq_exp,\n",
" include_input=include_input,\n",
")\n",
"\n",
"x_samples = torch.linspace(0, 1, resolution)\n",
"grid = torch.stack(torch.meshgrid([x_samples, x_samples], indexing=\"ij\"), dim=-1)\n",
"\n",
"encoded_values = encoder(grid)\n",
"\n",
"print(\"Input Values:\")\n",
"media.show_images(torch.moveaxis(grid, 2, 0), cmap=\"plasma\", border=True)\n",
"print(\"Encoded Values:\")\n",
"media.show_images(torch.moveaxis(encoded_values, 2, 0), vmin=-1, vmax=1, cmap=\"plasma\", border=True)\n",
"\n",
"print(\"Encoded Integrate Values:\")\n",
"for covariance_magnitude in covariance_magnitudes:\n",
" print(f\"Covariance Magnitude: {covariance_magnitude}\")\n",
" covs = torch.eye(2)[None, None, :, :] * covariance_magnitude\n",
" encoded_values = encoder(grid, covs=covs)\n",
" media.show_images(torch.moveaxis(encoded_values, 2, 0), vmin=-1, vmax=1, cmap=\"plasma\", border=True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(rff)=\n",
"## Random Fourier Feature (RFF) Encoding\n",
"This encoding assumes the inputs are between zero and one and can opperate on any dimensional input."
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"tags": [
"hide-input"
]
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Input Values:\n"
]
},
{
"data": {
"text/html": [
" | |
"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Encoded Values:\n"
]
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Encoded Integrate Values:\n",
"Covariance Magnitude: 0.001\n"
]
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Covariance Magnitude: 0.01\n"
]
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Covariance Magnitude: 0.1\n"
]
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# COLLAPSED\n",
"import torch\n",
"import mediapy as media\n",
"from nerfstudio.field_components import encoding\n",
"\n",
"num_frequencies = 8\n",
"scale = 10\n",
"resolution = 128\n",
"covariance_magnitudes = [0.001, 0.01, 0.1]\n",
"\n",
"encoder = encoding.RFFEncoding(in_dim=2, num_frequencies=num_frequencies, scale=scale)\n",
"\n",
"x_samples = torch.linspace(0, 1, resolution)\n",
"grid = torch.stack(torch.meshgrid([x_samples, x_samples], indexing=\"ij\"), dim=-1)\n",
"\n",
"encoded_values = encoder(grid)\n",
"\n",
"print(\"Input Values:\")\n",
"media.show_images(torch.moveaxis(grid, 2, 0), cmap=\"plasma\", border=True)\n",
"print(\"Encoded Values:\")\n",
"media.show_images(torch.moveaxis(encoded_values, 2, 0), cmap=\"plasma\", vmin=-1, vmax=1, border=True)\n",
"\n",
"print(\"Encoded Integrate Values:\")\n",
"for covariance_magnitude in covariance_magnitudes:\n",
" print(f\"Covariance Magnitude: {covariance_magnitude}\")\n",
" covs = torch.eye(2)[None, None, :, :] * covariance_magnitude\n",
" encoded_values = encoder(grid, covs=covs)\n",
" media.show_images(torch.moveaxis(encoded_values, 2, 0), cmap=\"plasma\", vmin=-1, vmax=1, border=True)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(hash)=\n",
"## Hash Encoding\n",
"The hash incoding was originally introduced in Instant-NGP. The encoding is optimized during training. This is a visualization of the initialization."
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {
"tags": [
"hide-input"
]
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"tensor(0.0010, grad_fn=)\n",
"tensor(-0.0010, grad_fn=)\n",
"Input Values:\n"
]
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Encoded Values:\n"
]
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# COLLAPSED\n",
"import torch\n",
"import mediapy as media\n",
"from nerfstudio.field_components import encoding\n",
"\n",
"num_levels = 8\n",
"min_res = 2\n",
"max_res = 128\n",
"log2_hashmap_size = 4 # Typically much larger tables are used\n",
"\n",
"resolution = 128\n",
"slice = 0\n",
"\n",
"# Fixing features_per_level to 3 for easy RGB visualization. Typical value is 2 in networks\n",
"features_per_level = 3\n",
"\n",
"encoder = encoding.HashEncoding(\n",
" num_levels=num_levels,\n",
" min_res=min_res,\n",
" max_res=max_res,\n",
" log2_hashmap_size=log2_hashmap_size,\n",
" features_per_level=features_per_level,\n",
" hash_init_scale=0.001,\n",
" implementation=\"torch\",\n",
")\n",
"\n",
"x_samples = torch.linspace(0, 1, resolution)\n",
"grid = torch.stack(torch.meshgrid([x_samples, x_samples, x_samples], indexing=\"ij\"), dim=-1)\n",
"\n",
"encoded_values = encoder(grid)\n",
"print(torch.max(encoded_values))\n",
"print(torch.min(encoded_values))\n",
"\n",
"grid_slice = grid[slice, ...]\n",
"encoded_values_slice = encoded_values[slice, ...]\n",
"\n",
"print(\"Input Values:\")\n",
"media.show_images(torch.moveaxis(grid_slice, 2, 0), cmap=\"plasma\", border=True)\n",
"\n",
"print(\"Encoded Values:\")\n",
"encoded_images = encoded_values_slice.view(resolution, resolution, num_levels, 3)\n",
"encoded_images = torch.moveaxis(encoded_images, 2, 0)\n",
"encoded_images -= torch.min(encoded_images)\n",
"encoded_images /= torch.max(encoded_images)\n",
"media.show_images(encoded_images.detach().numpy(), cmap=\"plasma\", border=True)"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"(spherical)=\n",
"## Spherical Harmonic Encoding\n",
"Encode direction using spherical harmonics. (Mostly used to encode viewing direction)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": [
"hide-input"
]
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Level: 1\n"
]
},
{
"data": {
"text/html": [
"![](\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAJYAAABkCAIAAADrOV6nAAABA0lEQVR4nO3RsQkAIQDAwPcrG+d2bYewkMDdBIGMNfdH2f86gFsW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYZ2GehXkW5lmYdwDkygFkzaTKPwAAAABJRU5ErkJggg==\"/) |
"
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Level: 2\n"
]
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Level: 3\n"
]
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Level: 4\n"
]
},
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# COLLAPSED\n",
"import torch\n",
"import mediapy as media\n",
"from nerfstudio.field_components import encoding\n",
"\n",
"levels = 4\n",
"\n",
"height = 100\n",
"width = 150\n",
"\n",
"encoder = encoding.SHEncoding(levels=levels)\n",
"\n",
"theta = torch.linspace(-torch.pi, torch.pi, width)\n",
"phi = torch.linspace(0, torch.pi, height)\n",
"[theta, phi] = torch.meshgrid([theta, phi], indexing=\"xy\")\n",
"\n",
"directions = torch.stack([torch.cos(theta) * torch.sin(phi), torch.sin(theta) * torch.sin(phi), torch.cos(phi)], dim=-1)\n",
"\n",
"encoded_values = encoder(directions)\n",
"encoded_values = torch.moveaxis(encoded_values, 2, 0)\n",
"\n",
"for level in range(levels):\n",
" print(f\"Level: {level+1}\")\n",
" media.show_images(encoded_values[level**2 : (level + 1) ** 2, ...], cmap=\"plasma\", border=True)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.8.12 ('nerfactory')",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.12"
},
"vscode": {
"interpreter": {
"hash": "34c28001ff35fb390494047002768a8182dcf55b1b11415165e62ea61557ab83"
}
}
},
"nbformat": 4,
"nbformat_minor": 4
}