Files
nerfstudio/nerfology/methods/feature_splatting.html
T
2025-07-29 02:30:47 +00:00

613 lines
36 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!doctype html>
<html class="no-js" lang="en" >
<head><meta charset="utf-8" />
<meta name="viewport" content="width=device-width,initial-scale=1" />
<meta name="color-scheme" content="light dark"><meta name="generator" content="Docutils 0.19: https://docutils.sourceforge.io/" />
<meta property="og:title" content="Feature Splatting" />
<meta property="og:type" content="website" />
<meta property="og:url" content="http://docs.nerf.studio/nerfology/methods/feature_splatting.html" />
<meta property="og:description" content="Paper Website Code Feature Splatting distills SAM-enhanced CLIP features into 3DGS for segmentation and editing Installation: First install nerfstudio dependencies. Then run: Running Feature Splatt..." />
<meta property="og:image" content="https://assets.nerf.studio/opg.png" />
<meta property="og:image:alt" content="Feature Splatting" />
<link rel="index" title="Index" href="../../genindex.html" /><link rel="search" title="Search" href="../../search.html" /><link rel="next" title="Mip-NeRF" href="mipnerf.html" /><link rel="prev" title="LiveScene" href="livescene.html" />
<meta name="generator" content="sphinx-5.2.1, furo 2022.09.29" />
<title>Feature Splatting - nerfstudio</title>
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="../../_static/styles/furo.css?digest=d81277517bee4d6b0349d71bb2661d4890b5617c" />
<link rel="stylesheet" type="text/css" href="../../_static/mystnb.css" />
<link rel="stylesheet" type="text/css" href="../../_static/togglebutton.css" />
<link rel="stylesheet" type="text/css" href="../../_static/copybutton.css" />
<link rel="stylesheet" type="text/css" href="../../_static/design-style.b7bb847fb20b106c3d81b95245e65545.min.css" />
<link rel="stylesheet" type="text/css" href="../../_static/styles/furo-extensions.css?digest=30d1aed668e5c3a91c3e3bf6a60b675221979f0e" />
<style>
body {
--color-code-background: #f8f9fb;
--color-code-foreground: black;
--color-brand-primary: #d34600;
--color-brand-content: #ff6f00;
}
@media not print {
body[data-theme="dark"] {
--color-code-background: #282C34;
--color-code-foreground: #ABB2BF;
--color-brand-primary: #fdd06c;
--color-brand-content: ##fea96a;
}
@media (prefers-color-scheme: dark) {
body:not([data-theme="light"]) {
--color-code-background: #282C34;
--color-code-foreground: #ABB2BF;
--color-brand-primary: #fdd06c;
--color-brand-content: ##fea96a;
}
}
}
</style><script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
<script src="../../_static/jquery.js"></script>
<script src="../../_static/underscore.js"></script>
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
<script src="../../_static/doctools.js"></script>
<script src="../../_static/sphinx_highlight.js"></script>
<script src="../../_static/scripts/furo.js"></script>
<script src="../../_static/clipboard.min.js"></script>
<script src="../../_static/copybutton.js"></script>
<script>let toggleHintShow = 'Click to show';</script>
<script>let toggleHintHide = 'Click to hide';</script>
<script>let toggleOpenOnPrint = 'true';</script>
<script src="../../_static/togglebutton.js"></script>
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown, .tag_hide_input div.cell_input, .tag_hide-input div.cell_input, .tag_hide_output div.cell_output, .tag_hide-output div.cell_output, .tag_hide_cell.cell, .tag_hide-cell.cell';</script>
<script src="../../_static/design-tabs.js"></script>
<script src="../../_static/require.min.js"></script>
<script src="../../_static/custom.js"></script>
</head>
<body>
<script>
document.body.dataset.theme = localStorage.getItem("theme") || "auto";
</script>
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
<symbol id="svg-toc" viewBox="0 0 24 24">
<title>Contents</title>
<svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 1024 1024">
<path d="M408 442h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8zm-8 204c0 4.4 3.6 8 8 8h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56zm504-486H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zm0 632H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zM115.4 518.9L271.7 642c5.8 4.6 14.4.5 14.4-6.9V388.9c0-7.4-8.5-11.5-14.4-6.9L115.4 505.1a8.74 8.74 0 0 0 0 13.8z"/>
</svg>
</symbol>
<symbol id="svg-menu" viewBox="0 0 24 24">
<title>Menu</title>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-menu">
<line x1="3" y1="12" x2="21" y2="12"></line>
<line x1="3" y1="6" x2="21" y2="6"></line>
<line x1="3" y1="18" x2="21" y2="18"></line>
</svg>
</symbol>
<symbol id="svg-arrow-right" viewBox="0 0 24 24">
<title>Expand</title>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-chevron-right">
<polyline points="9 18 15 12 9 6"></polyline>
</svg>
</symbol>
<symbol id="svg-sun" viewBox="0 0 24 24">
<title>Light mode</title>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" class="feather-sun">
<circle cx="12" cy="12" r="5"></circle>
<line x1="12" y1="1" x2="12" y2="3"></line>
<line x1="12" y1="21" x2="12" y2="23"></line>
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
<line x1="1" y1="12" x2="3" y2="12"></line>
<line x1="21" y1="12" x2="23" y2="12"></line>
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
</svg>
</symbol>
<symbol id="svg-moon" viewBox="0 0 24 24">
<title>Dark mode</title>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-moon">
<path stroke="none" d="M0 0h24v24H0z" fill="none" />
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z" />
</svg>
</symbol>
<symbol id="svg-sun-half" viewBox="0 0 24 24">
<title>Auto light/dark mode</title>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-shadow">
<path stroke="none" d="M0 0h24v24H0z" fill="none"/>
<circle cx="12" cy="12" r="9" />
<path d="M13 12h5" />
<path d="M13 15h4" />
<path d="M13 18h1" />
<path d="M13 9h4" />
<path d="M13 6h1" />
</svg>
</symbol>
</svg>
<input type="checkbox" class="sidebar-toggle" name="__navigation" id="__navigation">
<input type="checkbox" class="sidebar-toggle" name="__toc" id="__toc">
<label class="overlay sidebar-overlay" for="__navigation">
<div class="visually-hidden">Hide navigation sidebar</div>
</label>
<label class="overlay toc-overlay" for="__toc">
<div class="visually-hidden">Hide table of contents sidebar</div>
</label>
<div class="page">
<header class="mobile-header">
<div class="header-left">
<label class="nav-overlay-icon" for="__navigation">
<div class="visually-hidden">Toggle site navigation sidebar</div>
<i class="icon"><svg><use href="#svg-menu"></use></svg></i>
</label>
</div>
<div class="header-center">
<a href="../../index.html"><div class="brand">nerfstudio</div></a>
</div>
<div class="header-right">
<div class="theme-toggle-container theme-toggle-header">
<button class="theme-toggle">
<div class="visually-hidden">Toggle Light / Dark / Auto color theme</div>
<svg class="theme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
<svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
<svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
</button>
</div>
<label class="toc-overlay-icon toc-header-icon" for="__toc">
<div class="visually-hidden">Toggle table of contents sidebar</div>
<i class="icon"><svg><use href="#svg-toc"></use></svg></i>
</label>
</div>
</header>
<aside class="sidebar-drawer">
<div class="sidebar-container">
<div class="sidebar-sticky"><a
class="sidebar-brand"
href="../../index.html"
>
<div class="sidebar-logo-container">
<img
class="sidebar-logo only-light"
src="../../_static/imgs/logo.png"
alt="Light Logo"
/>
<img
class="sidebar-logo only-dark"
src="../../_static/imgs/logo-dark.png"
alt="Dark Logo"
/>
</div>
</a>
<div style="text-align: center">
<script async defer src="https://buttons.github.io/buttons.js"></script>
<!-- Place this tag where you want the button to render. -->
<a
class="github-button"
href="https://github.com/nerfstudio-project/nerfstudio/"
data-color-scheme="no-preference: light; light: light; dark: light;"
data-size="large"
data-show-count="true"
aria-label="Download buttons/github-buttons on GitHub"
>
Github
</a>
<br />
<a
href="https://colab.research.google.com/github/nerfstudio-project/nerfstudio/blob/main/colab/demo.ipynb"
style="
background-color: #ebf0f4;
background-image: linear-gradient(180deg, #f6f8fa, #ebf0f4 90%);
border-radius: 3px;
border: 1px solid #1b1f2426;
display: inline-flex;
cursor: pointer;
color: #0d0d0d;
font-family: Arial;
font-size: 12px;
font-weight: bold;
padding: 5px 41px;
line-height: 16px;
text-decoration: none;
"
>
<svg
viewBox="0 0 24 24"
width="16"
height="16"
class="octicon octicon-mark-github"
>
<g>
<path
d="M4.54,9.46,2.19,7.1a6.93,6.93,0,0,0,0,9.79l2.36-2.36A3.59,3.59,0,0,1,4.54,9.46Z"
style=""
fill="#E8710A"
></path>
<path
d="M2.19,7.1,4.54,9.46a3.59,3.59,0,0,1,5.08,0l1.71-2.93h0l-.1-.08h0A6.93,6.93,0,0,0,2.19,7.1Z"
style=""
fill="#F9AB00"
></path>
<path
d="M11.34,17.46h0L9.62,14.54a3.59,3.59,0,0,1-5.08,0L2.19,16.9a6.93,6.93,0,0,0,9,.65l.11-.09"
style=""
fill="#F9AB00"
></path>
<path
d="M12,7.1a6.93,6.93,0,0,0,0,9.79l2.36-2.36a3.59,3.59,0,1,1,5.08-5.08L21.81,7.1A6.93,6.93,0,0,0,12,7.1Z"
style=""
fill="#F9AB00"
></path>
<path
d="M21.81,7.1,19.46,9.46a3.59,3.59,0,0,1-5.08,5.08L12,16.9A6.93,6.93,0,0,0,21.81,7.1Z"
style=""
fill="#E8710A"
></path>
</g>
</svg>
<span>&nbspcolab</span></a
>
</div><form class="sidebar-search-container" method="get" action="../../search.html" role="search">
<input class="sidebar-search" placeholder=Search name="q" aria-label="Search">
<input type="hidden" name="check_keywords" value="yes">
<input type="hidden" name="area" value="default">
</form>
<div id="searchbox"></div><div class="sidebar-scroll"><div class="sidebar-tree">
<p class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../quickstart/installation.html">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../quickstart/first_nerf.html">Training your first model</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../quickstart/existing_dataset.html">Using existing data</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../quickstart/custom_dataset.html">Using custom data</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../quickstart/viewer_quickstart.html">Using the viewer</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../quickstart/export_geometry.html">Export geometry</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../quickstart/data_conventions.html">Data conventions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../reference/contributing.html">Contributing</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Extensions</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../extensions/blender_addon.html">Blender VFX add-on</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../extensions/maya_plugin.html">Autodesk Maya Plug-in</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../extensions/unreal_engine.html">Exporting to Unreal Engine</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../extensions/sdfstudio.html">SDFStudio</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">NeRFology</span></p>
<ul class="current">
<li class="toctree-l1 current has-children"><a class="reference internal" href="index.html">Methods</a><input checked="" class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" role="switch" type="checkbox"/><label for="toctree-checkbox-1"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="instant_ngp.html"> Instant-NGP</a></li>
<li class="toctree-l2"><a class="reference internal" href="splat.html"> Splatfacto</a></li>
<li class="toctree-l2"><a class="reference internal" href="splatw.html"> Splatfacto-W</a></li>
<li class="toctree-l2"><a class="reference internal" href="in2n.html"> Instruct-NeRF2NeRF</a></li>
<li class="toctree-l2"><a class="reference internal" href="igs2gs.html"> Instruct-GS2GS</a></li>
<li class="toctree-l2"><a class="reference internal" href="signerf.html"> SIGNeRF</a></li>
<li class="toctree-l2"><a class="reference internal" href="kplanes.html"> K-Planes</a></li>
<li class="toctree-l2"><a class="reference internal" href="lerf.html"> LERF</a></li>
<li class="toctree-l2"><a class="reference internal" href="livescene.html"> LiveScene</a></li>
<li class="toctree-l2 current current-page"><a class="current reference internal" href="#"> Feature-Splatting</a></li>
<li class="toctree-l2"><a class="reference internal" href="mipnerf.html"> Mip-NeRF</a></li>
<li class="toctree-l2"><a class="reference internal" href="nerf.html"> NeRF</a></li>
<li class="toctree-l2"><a class="reference internal" href="nerfacto.html"> Nerfacto</a></li>
<li class="toctree-l2"><a class="reference internal" href="nerfbusters.html"> Nerfbusters</a></li>
<li class="toctree-l2"><a class="reference internal" href="nerfplayer.html"> NeRFPlayer</a></li>
<li class="toctree-l2"><a class="reference internal" href="tetranerf.html"> Tetra-NeRF</a></li>
<li class="toctree-l2"><a class="reference internal" href="tensorf.html"> TensoRF</a></li>
<li class="toctree-l2"><a class="reference internal" href="generfacto.html"> Generfacto</a></li>
<li class="toctree-l2"><a class="reference internal" href="pynerf.html"> PyNeRF</a></li>
<li class="toctree-l2"><a class="reference internal" href="seathru_nerf.html"> SeaThru-NeRF</a></li>
<li class="toctree-l2"><a class="reference internal" href="zipnerf.html"> Zip-NeRF</a></li>
<li class="toctree-l2"><a class="reference internal" href="bionerf.html"> BioNeRF</a></li>
<li class="toctree-l2"><a class="reference internal" href="nerf2gs2nerf.html"> NeRFtoGSandBack</a></li>
<li class="toctree-l2"><a class="reference internal" href="opennerf.html"> OpenNeRF</a></li>
</ul>
</li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../model_components/index.html">Model components</a><input class="toctree-checkbox" id="toctree-checkbox-2" name="toctree-checkbox-2" role="switch" type="checkbox"/><label for="toctree-checkbox-2"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
<li class="toctree-l2"><a class="reference internal" href="../model_components/visualize_cameras.html"> Cameras models</a></li>
<li class="toctree-l2"><a class="reference internal" href="../model_components/visualize_samples.html"> Sample representation</a></li>
<li class="toctree-l2"><a class="reference internal" href="../model_components/visualize_samplers.html"> Ray samplers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../model_components/visualize_spatial_distortions.html"> Spatial distortions</a></li>
<li class="toctree-l2"><a class="reference internal" href="../model_components/visualize_encoders.html"> Encoders</a></li>
</ul>
</li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Developer Guides</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../developer_guides/new_methods.html">Adding a New Method</a></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../../developer_guides/pipelines/index.html">Pipelines overview</a><input class="toctree-checkbox" id="toctree-checkbox-3" name="toctree-checkbox-3" role="switch" type="checkbox"/><label for="toctree-checkbox-3"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
<li class="toctree-l2 has-children"><a class="reference internal" href="../../developer_guides/pipelines/dataparsers.html">DataParsers</a><input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" role="switch" type="checkbox"/><label for="toctree-checkbox-4"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/data/dataparsers.html">Data Parsers</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/pipelines/datamanagers.html">DataManagers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/pipelines/models.html">Models</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/pipelines/fields.html">Fields</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/pipelines/pipelines.html">Pipelines</a></li>
</ul>
</li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../../developer_guides/viewer/index.html">Viewer</a><input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" role="switch" type="checkbox"/><label for="toctree-checkbox-5"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/viewer/custom_gui.html">Custom GUI</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/viewer/viewer_control.html">Python Viewer Control</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/viewer/local_viewer.html">(Legacy Viewer) Local Server</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../developer_guides/config.html">Customizable configs</a></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../../developer_guides/debugging_tools/index.html">Debugging tools</a><input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" role="switch" type="checkbox"/><label for="toctree-checkbox-6"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/debugging_tools/local_logger.html">Local writer</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/debugging_tools/profiling.html">Code profiling support</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/debugging_tools/benchmarking.html">Benchmarking workflow</a></li>
</ul>
</li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Reference</span></p>
<ul>
<li class="toctree-l1 has-children"><a class="reference internal" href="../../reference/cli/index.html">CLI</a><input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" role="switch" type="checkbox"/><label for="toctree-checkbox-7"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_install_cli.html">ns-install-cli</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_process_data.html">ns-process-data</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_download_data.html">ns-download-data</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_train.html">ns-train</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_render.html">ns-render</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_viewer.html">ns-viewer</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_export.html">ns-export</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_eval.html">ns-eval</a></li>
</ul>
</li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../../reference/api/index.html">API</a><input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" role="switch" type="checkbox"/><label for="toctree-checkbox-8"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
<li class="toctree-l2"><a class="reference internal" href="../../reference/api/cameras.html">Cameras</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/api/config.html">Configs</a></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../../reference/api/data/index.html">Data</a><input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" role="switch" type="checkbox"/><label for="toctree-checkbox-9"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/data/dataparsers.html">Data Parsers</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/data/datamanagers.html">Datamanagers</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/data/datasets.html">Datasets</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/data/utils.html">Utils</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/api/fields.html">Fields</a></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../../reference/api/field_components/index.html">Field Modules</a><input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" role="switch" type="checkbox"/><label for="toctree-checkbox-10"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/field_components/encodings.html">Encodings</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/field_components/embeddings.html">Embeddings</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/field_components/field_heads.html">Field Heads</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/field_components/mlp.html">MLP</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/field_components/spatial_distortions.html">Spatial Distortions</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/api/models.html">Models</a></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../../reference/api/model_components/index.html">Model components</a><input class="toctree-checkbox" id="toctree-checkbox-11" name="toctree-checkbox-11" role="switch" type="checkbox"/><label for="toctree-checkbox-11"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/model_components/ray_sampler.html">Ray Sampler</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/model_components/losses.html">Losses</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/model_components/renderers.html">Renderers</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/api/optimizers.html">Engine</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/api/plugins.html">Plugins</a></li>
<li class="toctree-l2 has-children"><a class="reference internal" href="../../reference/api/utils/index.html">Utils</a><input class="toctree-checkbox" id="toctree-checkbox-12" name="toctree-checkbox-12" role="switch" type="checkbox"/><label for="toctree-checkbox-12"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/utils/colors.html">Colors</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/utils/math.html">Math</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/utils/colormaps.html">Colormaps</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/utils/tensor_dataclass.html">TensorDataclass</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../reference/api/viewer.html">Viewer</a></li>
</ul>
</li>
</ul>
</div>
</div>
</div>
</div>
</aside>
<div class="main">
<div class="content">
<div class="article-container">
<a href="#" class="back-to-top muted-link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24">
<path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12z"></path>
</svg>
<span>Back to top</span>
</a>
<div class="content-icon-container">
<div class="theme-toggle-container theme-toggle-content">
<button class="theme-toggle">
<div class="visually-hidden">Toggle Light / Dark / Auto color theme</div>
<svg class="theme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
<svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
<svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
</button>
</div>
<label class="toc-overlay-icon toc-content-icon" for="__toc">
<div class="visually-hidden">Toggle table of contents sidebar</div>
<i class="icon"><svg><use href="#svg-toc"></use></svg></i>
</label>
</div>
<article role="main">
<section class="tex2jax_ignore mathjax_ignore" id="feature-splatting">
<h1>Feature Splatting<a class="headerlink" href="#feature-splatting" title="Permalink to this heading">#</a></h1>
<h4>Feature Splatting</h4>
<p><a class="sd-sphinx-override sd-btn sd-text-wrap sd-btn-outline-primary reference external" href="https://feature-splatting.github.io/"><span>Paper Website</span></a></p>
<p><a class="sd-sphinx-override sd-btn sd-text-wrap sd-btn-outline-primary reference external" href="https://github.com/vuer-ai/feature-splatting/"><span>Code</span></a></p>
<video id="teaser" muted autoplay playsinline loop controls width="100%">
<source id="mp4" src="https://feature-splatting.github.io/resources/basic_ns_demo_feature_only.mp4" type="video/mp4">
</video>
<p><strong>Feature Splatting distills SAM-enhanced CLIP features into 3DGS for segmentation and editing</strong></p>
<section id="installation">
<h2>Installation<a class="headerlink" href="#installation" title="Permalink to this heading">#</a></h2>
<p>First install nerfstudio dependencies. Then run:</p>
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>pip<span class="w"> </span>install<span class="w"> </span>git+https://github.com/vuer-ai/feature-splatting
</pre></div>
</div>
</section>
<section id="running-feature-splatting">
<h2>Running Feature Splatting<a class="headerlink" href="#running-feature-splatting" title="Permalink to this heading">#</a></h2>
<p>Details for running Feature Splatting (built with Nerfstudio!) can be found <a class="reference external" href="https://github.com/vuer-ai/feature-splatting">here</a>.
Once installed, run:</p>
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>ns-train<span class="w"> </span>feature-splatting<span class="w"> </span>--help
</pre></div>
</div>
<p>Currently, we provide the following variants:</p>
<div class="table-wrapper colwidths-auto docutils container">
<table class="docutils align-default">
<thead>
<tr class="row-odd"><th class="head"><p>Method</p></th>
<th class="head"><p>Description</p></th>
<th class="head"><p>Memory</p></th>
<th class="head"><p>Quality</p></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">feature-splatting</span></code></p></td>
<td><p>Feature Splatting with MaskCLIP ViT-L/14&#64;336px and MobileSAMv2</p></td>
<td><p>~8 GB</p></td>
<td><p>Good</p></td>
</tr>
</tbody>
</table>
</div>
<p>Note that the reference features used in this version are different from the version used in the paper in two ways</p>
<ul class="simple">
<li><p>The SAM-enhanced CLIP features are computed using MobileSAMv2, which is much faster than original SAM but slightly less accurate.</p></li>
<li><p>The CLIP features are computed only on the image-level.</p></li>
</ul>
</section>
<section id="method">
<h2>Method<a class="headerlink" href="#method" title="Permalink to this heading">#</a></h2>
<p>Feature splatting distills CLIP features into 3DGS by view-independent rasterization, which allows open-vocabulary 2D segmentation and open-vocabulary 3D segmentation of Gaussians directly in the 3D space. This implementation supports simple editing applications by directly manipulating Gaussians.</p>
<section id="reference-feature-computation-and-joint-supervision">
<h3>Reference feature computation and joint supervision<a class="headerlink" href="#reference-feature-computation-and-joint-supervision" title="Permalink to this heading">#</a></h3>
<p>Feature splatting computes high-quality SAM-enhanced CLIP features as reference features. Compared to coarse CLIP features (such as those used in LERF), Feature splatting performs an object-level masked average pooling of the features to refine the boundary of objects. While the original ECCV’24 paper uses SAM for part-level masks, this implementation uses MobileSAMv2 for much faster reference features computation, which we hope would encourage downstream applications that require real-time performance.</p>
<p>In addition to SAM-enhanced features, we also found that using DINOv2 features as a joint supervision helps regularize the internal structure of objects, which is similar to findings in existing work.</p>
</section>
<section id="scene-editing">
<h3>Scene Editing<a class="headerlink" href="#scene-editing" title="Permalink to this heading">#</a></h3>
<p>Thanks to the explicit representation of 3DGS, grouped Gaussians can be easily manipulated. While the original ECCV’24 paper proposes a series of editing primitives, to avoid introducing excessive dependencies or hacks, we support a subset of editing primitives in this implementation:</p>
<p>Rigid operations</p>
<ul class="simple">
<li><p>Floor estimation (for intuitive rotation and gravity estimation)</p></li>
<li><p>Translation</p></li>
<li><p>Transparent (highlights segmented object and turns background Gaussians transparent)</p></li>
<li><p>Rotation (yaw only w.r.t. estimated ground)</p></li>
</ul>
<p>Non-rigid operations</p>
<ul class="simple">
<li><p>Sand-like melting (based on Taichi MPM method)</p></li>
</ul>
<video id="teaser" muted autoplay playsinline loop controls width="100%">
<source id="mp4" src="https://feature-splatting.github.io/resources/ns_editing_compressed.mp4" type="video/mp4">
</video>
<p>If you find our work helpful for your research, please consider citing</p>
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>@inproceedings{qiu-2024-featuresplatting,
title={Language-Driven Physics-Based Scene Synthesis and Editing via Feature Splatting},
author={Ri-Zhao Qiu and Ge Yang and Weijia Zeng and Xiaolong Wang},
booktitle={European Conference on Computer Vision (ECCV)},
year={2024}
}
</pre></div>
</div>
</section>
</section>
</section>
</article>
</div>
<footer>
<div class="related-pages">
<a class="next-page" href="mipnerf.html">
<div class="page-info">
<div class="context">
<span>Next</span>
</div>
<div class="title">Mip-NeRF</div>
</div>
<svg class="furo-related-icon"><use href="#svg-arrow-right"></use></svg>
</a>
<a class="prev-page" href="livescene.html">
<svg class="furo-related-icon"><use href="#svg-arrow-right"></use></svg>
<div class="page-info">
<div class="context">
<span>Previous</span>
</div>
<div class="title">LiveScene</div>
</div>
</a>
</div>
<div class="bottom-of-page">
<div class="left-details">
<div class="copyright">
Copyright &#169; 2022, nerfstudio Team
</div>
Made with <a href="https://www.sphinx-doc.org/">Sphinx</a> and <a class="muted-link" href="https://pradyunsg.me">@pradyunsg</a>'s
<a href="https://github.com/pradyunsg/furo">Furo</a>
</div>
<div class="right-details">
<div class="icons">
</div>
</div>
</div>
</footer>
</div>
<aside class="toc-drawer">
<div class="toc-sticky toc-scroll">
<div class="toc-title-container">
<span class="toc-title">
On this page
</span>
</div>
<div class="toc-tree-container">
<div class="toc-tree">
<ul>
<li><a class="reference internal" href="#">Feature Splatting</a><ul>
<li><a class="reference internal" href="#installation">Installation</a></li>
<li><a class="reference internal" href="#running-feature-splatting">Running Feature Splatting</a></li>
<li><a class="reference internal" href="#method">Method</a><ul>
<li><a class="reference internal" href="#reference-feature-computation-and-joint-supervision">Reference feature computation and joint supervision</a></li>
<li><a class="reference internal" href="#scene-editing">Scene Editing</a></li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
</div>
</div>
</aside>
</div>
</div></body>
</html>