{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Visualize Field Encoders\n" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "application/javascript": "\n setTimeout(function() {\n var nbb_cell_id = 6;\n var nbb_unformatted_code = \"import torch\\nimport mediapy as media\\nfrom mattport.nerf.field_modules import encoding\";\n var nbb_formatted_code = \"import torch\\nimport mediapy as media\\nfrom mattport.nerf.field_modules import encoding\";\n var nbb_cells = Jupyter.notebook.get_cells();\n for (var i = 0; i < nbb_cells.length; ++i) {\n if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n nbb_cells[i].set_text(nbb_formatted_code);\n }\n break;\n }\n }\n }, 500);\n ", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import torch\n", "import mediapy as media\n", "from mattport.nerf.field_modules import encoding" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# NeRF Positional Encoding\n", "First introduced in the original NeRF paper. This encoding assumes the inputs are between zero and one and can opperate on any dimensional input." ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Input Values:\n" ] }, { "data": { "text/html": [ "
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Encoded Values:\n" ] }, { "data": { "text/html": [ "
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/javascript": "\n setTimeout(function() {\n var nbb_cell_id = 7;\n var nbb_unformatted_code = \"num_frequencies = 4\\nmin_freq_exp = 0\\nmax_freq_exp = 6\\ninclude_input = True\\nresolution = 128\\n\\nencoder = encoding.NeRFEncoding(\\n in_dim=2,\\n num_frequencies=num_frequencies,\\n min_freq_exp=min_freq_exp,\\n max_freq_exp=max_freq_exp,\\n include_input=include_input,\\n)\\n\\nx_samples = torch.linspace(0, 1, resolution)\\ngrid = torch.stack(torch.meshgrid([x_samples, x_samples], indexing=\\\"ij\\\"), dim=-1)\\n\\nencoded_values = encoder(grid)\\n\\nprint(\\\"Input Values:\\\")\\nmedia.show_images(torch.moveaxis(grid, 2, 0), cmap=\\\"plasma\\\", border=True)\\nprint(\\\"Encoded Values:\\\")\\nmedia.show_images(torch.moveaxis(encoded_values, 2, 0), cmap=\\\"plasma\\\", border=True)\";\n var nbb_formatted_code = \"num_frequencies = 4\\nmin_freq_exp = 0\\nmax_freq_exp = 6\\ninclude_input = True\\nresolution = 128\\n\\nencoder = encoding.NeRFEncoding(\\n in_dim=2,\\n num_frequencies=num_frequencies,\\n min_freq_exp=min_freq_exp,\\n max_freq_exp=max_freq_exp,\\n include_input=include_input,\\n)\\n\\nx_samples = torch.linspace(0, 1, resolution)\\ngrid = torch.stack(torch.meshgrid([x_samples, x_samples], indexing=\\\"ij\\\"), dim=-1)\\n\\nencoded_values = encoder(grid)\\n\\nprint(\\\"Input Values:\\\")\\nmedia.show_images(torch.moveaxis(grid, 2, 0), cmap=\\\"plasma\\\", border=True)\\nprint(\\\"Encoded Values:\\\")\\nmedia.show_images(torch.moveaxis(encoded_values, 2, 0), cmap=\\\"plasma\\\", border=True)\";\n var nbb_cells = Jupyter.notebook.get_cells();\n for (var i = 0; i < nbb_cells.length; ++i) {\n if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n nbb_cells[i].set_text(nbb_formatted_code);\n }\n break;\n }\n }\n }, 500);\n ", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "num_frequencies = 4\n", "min_freq_exp = 0\n", "max_freq_exp = 6\n", "include_input = True\n", "resolution = 128\n", "\n", "encoder = encoding.NeRFEncoding(\n", " in_dim=2,\n", " num_frequencies=num_frequencies,\n", " min_freq_exp=min_freq_exp,\n", " max_freq_exp=max_freq_exp,\n", " include_input=include_input,\n", ")\n", "\n", "x_samples = torch.linspace(0, 1, resolution)\n", "grid = torch.stack(torch.meshgrid([x_samples, x_samples], indexing=\"ij\"), dim=-1)\n", "\n", "encoded_values = encoder(grid)\n", "\n", "print(\"Input Values:\")\n", "media.show_images(torch.moveaxis(grid, 2, 0), cmap=\"plasma\", border=True)\n", "print(\"Encoded Values:\")\n", "media.show_images(torch.moveaxis(encoded_values, 2, 0), cmap=\"plasma\", border=True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Random Fourier Feature (RFF) Encoding\n", "This encoding assumes the inputs are between zero and one and can opperate on any dimensional input." ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Input Values:\n" ] }, { "data": { "text/html": [ "
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Encoded Values:\n" ] }, { "data": { "text/html": [ "
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/javascript": "\n setTimeout(function() {\n var nbb_cell_id = 8;\n var nbb_unformatted_code = \"num_frequencies = 8\\nscale = 10\\nresolution = 128\\n\\nencoder = encoding.RFFEncoding(in_dim=2, num_frequencies=num_frequencies, scale=scale)\\n\\nx_samples = torch.linspace(0, 1, resolution)\\ngrid = torch.stack(torch.meshgrid([x_samples, x_samples], indexing=\\\"ij\\\"), dim=-1)\\n\\nencoded_values = encoder(grid)\\n\\nprint(\\\"Input Values:\\\")\\nmedia.show_images(torch.moveaxis(grid, 2, 0), cmap=\\\"plasma\\\", border=True)\\nprint(\\\"Encoded Values:\\\")\\nmedia.show_images(torch.moveaxis(encoded_values, 2, 0), cmap=\\\"plasma\\\", border=True)\";\n var nbb_formatted_code = \"num_frequencies = 8\\nscale = 10\\nresolution = 128\\n\\nencoder = encoding.RFFEncoding(in_dim=2, num_frequencies=num_frequencies, scale=scale)\\n\\nx_samples = torch.linspace(0, 1, resolution)\\ngrid = torch.stack(torch.meshgrid([x_samples, x_samples], indexing=\\\"ij\\\"), dim=-1)\\n\\nencoded_values = encoder(grid)\\n\\nprint(\\\"Input Values:\\\")\\nmedia.show_images(torch.moveaxis(grid, 2, 0), cmap=\\\"plasma\\\", border=True)\\nprint(\\\"Encoded Values:\\\")\\nmedia.show_images(torch.moveaxis(encoded_values, 2, 0), cmap=\\\"plasma\\\", border=True)\";\n var nbb_cells = Jupyter.notebook.get_cells();\n for (var i = 0; i < nbb_cells.length; ++i) {\n if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n nbb_cells[i].set_text(nbb_formatted_code);\n }\n break;\n }\n }\n }, 500);\n ", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "num_frequencies = 8\n", "scale = 10\n", "resolution = 128\n", "\n", "encoder = encoding.RFFEncoding(in_dim=2, num_frequencies=num_frequencies, scale=scale)\n", "\n", "x_samples = torch.linspace(0, 1, resolution)\n", "grid = torch.stack(torch.meshgrid([x_samples, x_samples], indexing=\"ij\"), dim=-1)\n", "\n", "encoded_values = encoder(grid)\n", "\n", "print(\"Input Values:\")\n", "media.show_images(torch.moveaxis(grid, 2, 0), cmap=\"plasma\", border=True)\n", "print(\"Encoded Values:\")\n", "media.show_images(torch.moveaxis(encoded_values, 2, 0), cmap=\"plasma\", border=True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Hash Encoding\n", "The hash incoding was originally introduced in Instant-NGP. The encoding is optimized during training. This is a visualization of the initialization." ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Input Values:\n" ] }, { "data": { "text/html": [ "
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Encoded Values:\n" ] }, { "data": { "text/html": [ "
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/javascript": "\n setTimeout(function() {\n var nbb_cell_id = 9;\n var nbb_unformatted_code = \"num_levels = 8\\nmin_res = 2\\nmax_res = 128\\nhash_table_size = 2**4 # Typically much larger tables are used\\n\\nresolution = 128\\nslice = 0\\n\\n# Fixing features_per_level to 3 for easy RGB visualization. Typical value is 2 in networks\\nfeatures_per_level = 3\\n\\nencoder = encoding.HashEncoding(\\n num_levels=num_levels,\\n min_res=min_res,\\n max_res=max_res,\\n hash_table_size=hash_table_size,\\n features_per_level=features_per_level,\\n hash_init_scale=0.001,\\n)\\n\\nx_samples = torch.linspace(0, 1, resolution)\\ngrid = torch.stack(torch.meshgrid([x_samples, x_samples, x_samples], indexing=\\\"ij\\\"), dim=-1)\\n\\nencoded_values = encoder(grid)\\n\\ngrid_slice = grid[slice, ...]\\nencoded_values_slice = encoded_values[slice, ...]\\n\\nprint(\\\"Input Values:\\\")\\nmedia.show_images(torch.moveaxis(grid_slice, 2, 0), cmap=\\\"plasma\\\", border=True)\\n\\nprint(\\\"Encoded Values:\\\")\\nencoded_images = encoded_values_slice.view(resolution, resolution, num_levels, 3)\\nencoded_images = torch.moveaxis(encoded_images, 2, 0)\\nencoded_images -= torch.min(encoded_images)\\nencoded_images /= torch.max(encoded_images)\\nmedia.show_images(encoded_images.detach().numpy(), cmap=\\\"plasma\\\", border=True)\";\n var nbb_formatted_code = \"num_levels = 8\\nmin_res = 2\\nmax_res = 128\\nhash_table_size = 2**4 # Typically much larger tables are used\\n\\nresolution = 128\\nslice = 0\\n\\n# Fixing features_per_level to 3 for easy RGB visualization. Typical value is 2 in networks\\nfeatures_per_level = 3\\n\\nencoder = encoding.HashEncoding(\\n num_levels=num_levels,\\n min_res=min_res,\\n max_res=max_res,\\n hash_table_size=hash_table_size,\\n features_per_level=features_per_level,\\n hash_init_scale=0.001,\\n)\\n\\nx_samples = torch.linspace(0, 1, resolution)\\ngrid = torch.stack(\\n torch.meshgrid([x_samples, x_samples, x_samples], indexing=\\\"ij\\\"), dim=-1\\n)\\n\\nencoded_values = encoder(grid)\\n\\ngrid_slice = grid[slice, ...]\\nencoded_values_slice = encoded_values[slice, ...]\\n\\nprint(\\\"Input Values:\\\")\\nmedia.show_images(torch.moveaxis(grid_slice, 2, 0), cmap=\\\"plasma\\\", border=True)\\n\\nprint(\\\"Encoded Values:\\\")\\nencoded_images = encoded_values_slice.view(resolution, resolution, num_levels, 3)\\nencoded_images = torch.moveaxis(encoded_images, 2, 0)\\nencoded_images -= torch.min(encoded_images)\\nencoded_images /= torch.max(encoded_images)\\nmedia.show_images(encoded_images.detach().numpy(), cmap=\\\"plasma\\\", border=True)\";\n var nbb_cells = Jupyter.notebook.get_cells();\n for (var i = 0; i < nbb_cells.length; ++i) {\n if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n nbb_cells[i].set_text(nbb_formatted_code);\n }\n break;\n }\n }\n }, 500);\n ", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "num_levels = 8\n", "min_res = 2\n", "max_res = 128\n", "hash_table_size = 2**4 # Typically much larger tables are used\n", "\n", "resolution = 128\n", "slice = 0\n", "\n", "# Fixing features_per_level to 3 for easy RGB visualization. Typical value is 2 in networks\n", "features_per_level = 3\n", "\n", "encoder = encoding.HashEncoding(\n", " num_levels=num_levels,\n", " min_res=min_res,\n", " max_res=max_res,\n", " hash_table_size=hash_table_size,\n", " features_per_level=features_per_level,\n", " hash_init_scale=0.001,\n", ")\n", "\n", "x_samples = torch.linspace(0, 1, resolution)\n", "grid = torch.stack(torch.meshgrid([x_samples, x_samples, x_samples], indexing=\"ij\"), dim=-1)\n", "\n", "encoded_values = encoder(grid)\n", "\n", "grid_slice = grid[slice, ...]\n", "encoded_values_slice = encoded_values[slice, ...]\n", "\n", "print(\"Input Values:\")\n", "media.show_images(torch.moveaxis(grid_slice, 2, 0), cmap=\"plasma\", border=True)\n", "\n", "print(\"Encoded Values:\")\n", "encoded_images = encoded_values_slice.view(resolution, resolution, num_levels, 3)\n", "encoded_images = torch.moveaxis(encoded_images, 2, 0)\n", "encoded_images -= torch.min(encoded_images)\n", "encoded_images /= torch.max(encoded_images)\n", "media.show_images(encoded_images.detach().numpy(), cmap=\"plasma\", border=True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Spherical Harmonic Encoding\n", "Encode direction using spherical harmoincs. (Mostly used to encode viewing direction)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Level: 1\n" ] }, { "data": { "text/html": [ "
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Level: 2\n" ] }, { "data": { "text/html": [ "
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Level: 3\n" ] }, { "data": { "text/html": [ "
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Level: 4\n" ] }, { "data": { "text/html": [ "
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/javascript": "\n setTimeout(function() {\n var nbb_cell_id = 10;\n var nbb_unformatted_code = \"levels = 4\\n\\nheight = 100\\nwidth = 150\\n\\nencoder = encoding.SHEncoding(levels=levels)\\n\\ntheta = torch.linspace(-torch.pi, torch.pi, width)\\nphi = torch.linspace(0, torch.pi, height)\\n[theta, phi] = torch.meshgrid([theta, phi], indexing=\\\"xy\\\")\\n\\ndirections = torch.stack([torch.cos(theta) * torch.sin(phi), torch.sin(theta) * torch.sin(phi), torch.cos(phi)], dim=-1)\\n\\nencoded_values = encoder(directions)\\nencoded_values = torch.moveaxis(encoded_values, 2, 0)\\n\\nfor level in range(levels):\\n print(f\\\"Level: {level+1}\\\")\\n media.show_images(encoded_values[level**2 : (level + 1) ** 2, ...], cmap=\\\"plasma\\\", border=True)\";\n var nbb_formatted_code = \"levels = 4\\n\\nheight = 100\\nwidth = 150\\n\\nencoder = encoding.SHEncoding(levels=levels)\\n\\ntheta = torch.linspace(-torch.pi, torch.pi, width)\\nphi = torch.linspace(0, torch.pi, height)\\n[theta, phi] = torch.meshgrid([theta, phi], indexing=\\\"xy\\\")\\n\\ndirections = torch.stack(\\n [\\n torch.cos(theta) * torch.sin(phi),\\n torch.sin(theta) * torch.sin(phi),\\n torch.cos(phi),\\n ],\\n dim=-1,\\n)\\n\\nencoded_values = encoder(directions)\\nencoded_values = torch.moveaxis(encoded_values, 2, 0)\\n\\nfor level in range(levels):\\n print(f\\\"Level: {level+1}\\\")\\n media.show_images(\\n encoded_values[level**2 : (level + 1) ** 2, ...], cmap=\\\"plasma\\\", border=True\\n )\";\n var nbb_cells = Jupyter.notebook.get_cells();\n for (var i = 0; i < nbb_cells.length; ++i) {\n if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n nbb_cells[i].set_text(nbb_formatted_code);\n }\n break;\n }\n }\n }, 500);\n ", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "levels = 4\n", "\n", "height = 100\n", "width = 150\n", "\n", "encoder = encoding.SHEncoding(levels=levels)\n", "\n", "theta = torch.linspace(-torch.pi, torch.pi, width)\n", "phi = torch.linspace(0, torch.pi, height)\n", "[theta, phi] = torch.meshgrid([theta, phi], indexing=\"xy\")\n", "\n", "directions = torch.stack([torch.cos(theta) * torch.sin(phi), torch.sin(theta) * torch.sin(phi), torch.cos(phi)], dim=-1)\n", "\n", "encoded_values = encoder(directions)\n", "encoded_values = torch.moveaxis(encoded_values, 2, 0)\n", "\n", "for level in range(levels):\n", " print(f\"Level: {level+1}\")\n", " media.show_images(encoded_values[level**2 : (level + 1) ** 2, ...], cmap=\"plasma\", border=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "interpreter": { "hash": "024fe9dea615e4e34961116ff517c7e9f5910440e3d2c8eb4560299f11aaf2e7" }, "kernelspec": { "display_name": "Python 3.9.11 ('mattport')", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.12" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }