mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-12 19:04:03 +08:00
573 lines
37 KiB
HTML
573 lines
37 KiB
HTML
<!doctype html>
|
||
<html class="no-js" lang="en" >
|
||
|
||
<head><meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width,initial-scale=1" />
|
||
<meta name="color-scheme" content="light dark"><meta name="generator" content="Docutils 0.19: https://docutils.sourceforge.io/" />
|
||
|
||
<meta property="og:title" content="OpenNeRF" />
|
||
|
||
<meta property="og:type" content="website" />
|
||
|
||
<meta property="og:url" content="http://docs.nerf.studio/nerfology/methods/opennerf.html" />
|
||
|
||
<meta property="og:description" content="OpenSet 3D Neural Scene Segmentation with Pixel-wise Features and Rendered Novel Views: Paper Website Installation: Detailed installation instructions can be found at our Project Page. Running Open..." />
|
||
|
||
<meta property="og:image" content="https://assets.nerf.studio/opg.png" />
|
||
|
||
<meta property="og:image:alt" content="OpenNeRF" />
|
||
<link rel="index" title="Index" href="../../genindex.html" /><link rel="search" title="Search" href="../../search.html" /><link rel="next" title="Model components" href="../model_components/index.html" /><link rel="prev" title="From NeRFs to Gaussian Splats, and Back" href="nerf2gs2nerf.html" />
|
||
|
||
<meta name="generator" content="sphinx-5.2.1, furo 2022.09.29" />
|
||
<title>OpenNeRF - nerfstudio</title>
|
||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||
<link rel="stylesheet" type="text/css" href="../../_static/styles/furo.css?digest=d81277517bee4d6b0349d71bb2661d4890b5617c" />
|
||
<link rel="stylesheet" type="text/css" href="../../_static/mystnb.css" />
|
||
<link rel="stylesheet" type="text/css" href="../../_static/togglebutton.css" />
|
||
<link rel="stylesheet" type="text/css" href="../../_static/copybutton.css" />
|
||
<link rel="stylesheet" type="text/css" href="../../_static/design-style.b7bb847fb20b106c3d81b95245e65545.min.css" />
|
||
<link rel="stylesheet" type="text/css" href="../../_static/styles/furo-extensions.css?digest=30d1aed668e5c3a91c3e3bf6a60b675221979f0e" />
|
||
|
||
|
||
|
||
|
||
<style>
|
||
body {
|
||
--color-code-background: #f8f9fb;
|
||
--color-code-foreground: black;
|
||
--color-brand-primary: #d34600;
|
||
--color-brand-content: #ff6f00;
|
||
|
||
}
|
||
@media not print {
|
||
body[data-theme="dark"] {
|
||
--color-code-background: #282C34;
|
||
--color-code-foreground: #ABB2BF;
|
||
--color-brand-primary: #fdd06c;
|
||
--color-brand-content: ##fea96a;
|
||
|
||
}
|
||
@media (prefers-color-scheme: dark) {
|
||
body:not([data-theme="light"]) {
|
||
--color-code-background: #282C34;
|
||
--color-code-foreground: #ABB2BF;
|
||
--color-brand-primary: #fdd06c;
|
||
--color-brand-content: ##fea96a;
|
||
|
||
}
|
||
}
|
||
}
|
||
</style><script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||
<script src="../../_static/jquery.js"></script>
|
||
<script src="../../_static/underscore.js"></script>
|
||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||
<script src="../../_static/doctools.js"></script>
|
||
<script src="../../_static/sphinx_highlight.js"></script>
|
||
<script src="../../_static/scripts/furo.js"></script>
|
||
<script src="../../_static/clipboard.min.js"></script>
|
||
<script src="../../_static/copybutton.js"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="../../_static/togglebutton.js"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown, .tag_hide_input div.cell_input, .tag_hide-input div.cell_input, .tag_hide_output div.cell_output, .tag_hide-output div.cell_output, .tag_hide_cell.cell, .tag_hide-cell.cell';</script>
|
||
<script src="../../_static/design-tabs.js"></script>
|
||
<script src="../../_static/require.min.js"></script>
|
||
<script src="../../_static/custom.js"></script>
|
||
</head>
|
||
|
||
<body>
|
||
|
||
<script>
|
||
document.body.dataset.theme = localStorage.getItem("theme") || "auto";
|
||
</script>
|
||
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
|
||
<symbol id="svg-toc" viewBox="0 0 24 24">
|
||
<title>Contents</title>
|
||
<svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 1024 1024">
|
||
<path d="M408 442h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8zm-8 204c0 4.4 3.6 8 8 8h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56zm504-486H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zm0 632H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zM115.4 518.9L271.7 642c5.8 4.6 14.4.5 14.4-6.9V388.9c0-7.4-8.5-11.5-14.4-6.9L115.4 505.1a8.74 8.74 0 0 0 0 13.8z"/>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-menu" viewBox="0 0 24 24">
|
||
<title>Menu</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
|
||
stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-menu">
|
||
<line x1="3" y1="12" x2="21" y2="12"></line>
|
||
<line x1="3" y1="6" x2="21" y2="6"></line>
|
||
<line x1="3" y1="18" x2="21" y2="18"></line>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-arrow-right" viewBox="0 0 24 24">
|
||
<title>Expand</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
|
||
stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-chevron-right">
|
||
<polyline points="9 18 15 12 9 6"></polyline>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun" viewBox="0 0 24 24">
|
||
<title>Light mode</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
|
||
stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" class="feather-sun">
|
||
<circle cx="12" cy="12" r="5"></circle>
|
||
<line x1="12" y1="1" x2="12" y2="3"></line>
|
||
<line x1="12" y1="21" x2="12" y2="23"></line>
|
||
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
|
||
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
|
||
<line x1="1" y1="12" x2="3" y2="12"></line>
|
||
<line x1="21" y1="12" x2="23" y2="12"></line>
|
||
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
|
||
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-moon" viewBox="0 0 24 24">
|
||
<title>Dark mode</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
|
||
stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-moon">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none" />
|
||
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z" />
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun-half" viewBox="0 0 24 24">
|
||
<title>Auto light/dark mode</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
|
||
stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-shadow">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none"/>
|
||
<circle cx="12" cy="12" r="9" />
|
||
<path d="M13 12h5" />
|
||
<path d="M13 15h4" />
|
||
<path d="M13 18h1" />
|
||
<path d="M13 9h4" />
|
||
<path d="M13 6h1" />
|
||
</svg>
|
||
</symbol>
|
||
</svg>
|
||
|
||
<input type="checkbox" class="sidebar-toggle" name="__navigation" id="__navigation">
|
||
<input type="checkbox" class="sidebar-toggle" name="__toc" id="__toc">
|
||
<label class="overlay sidebar-overlay" for="__navigation">
|
||
<div class="visually-hidden">Hide navigation sidebar</div>
|
||
</label>
|
||
<label class="overlay toc-overlay" for="__toc">
|
||
<div class="visually-hidden">Hide table of contents sidebar</div>
|
||
</label>
|
||
|
||
|
||
|
||
<div class="page">
|
||
<header class="mobile-header">
|
||
<div class="header-left">
|
||
<label class="nav-overlay-icon" for="__navigation">
|
||
<div class="visually-hidden">Toggle site navigation sidebar</div>
|
||
<i class="icon"><svg><use href="#svg-menu"></use></svg></i>
|
||
</label>
|
||
</div>
|
||
<div class="header-center">
|
||
<a href="../../index.html"><div class="brand">nerfstudio</div></a>
|
||
</div>
|
||
<div class="header-right">
|
||
<div class="theme-toggle-container theme-toggle-header">
|
||
<button class="theme-toggle">
|
||
<div class="visually-hidden">Toggle Light / Dark / Auto color theme</div>
|
||
<svg class="theme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
|
||
<svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
</button>
|
||
</div>
|
||
<label class="toc-overlay-icon toc-header-icon" for="__toc">
|
||
<div class="visually-hidden">Toggle table of contents sidebar</div>
|
||
<i class="icon"><svg><use href="#svg-toc"></use></svg></i>
|
||
</label>
|
||
</div>
|
||
</header>
|
||
<aside class="sidebar-drawer">
|
||
<div class="sidebar-container">
|
||
|
||
<div class="sidebar-sticky"><a
|
||
class="sidebar-brand"
|
||
href="../../index.html"
|
||
>
|
||
|
||
<div class="sidebar-logo-container">
|
||
<img
|
||
class="sidebar-logo only-light"
|
||
src="../../_static/imgs/logo.png"
|
||
alt="Light Logo"
|
||
/>
|
||
<img
|
||
class="sidebar-logo only-dark"
|
||
src="../../_static/imgs/logo-dark.png"
|
||
alt="Dark Logo"
|
||
/>
|
||
</div>
|
||
</a>
|
||
|
||
<div style="text-align: center">
|
||
<script async defer src="https://buttons.github.io/buttons.js"></script>
|
||
<!-- Place this tag where you want the button to render. -->
|
||
<a
|
||
class="github-button"
|
||
href="https://github.com/nerfstudio-project/nerfstudio/"
|
||
data-color-scheme="no-preference: light; light: light; dark: light;"
|
||
data-size="large"
|
||
data-show-count="true"
|
||
aria-label="Download buttons/github-buttons on GitHub"
|
||
>
|
||
Github
|
||
</a>
|
||
<br />
|
||
<a
|
||
href="https://colab.research.google.com/github/nerfstudio-project/nerfstudio/blob/main/colab/demo.ipynb"
|
||
style="
|
||
background-color: #ebf0f4;
|
||
background-image: linear-gradient(180deg, #f6f8fa, #ebf0f4 90%);
|
||
border-radius: 3px;
|
||
border: 1px solid #1b1f2426;
|
||
display: inline-flex;
|
||
cursor: pointer;
|
||
color: #0d0d0d;
|
||
font-family: Arial;
|
||
font-size: 12px;
|
||
font-weight: bold;
|
||
padding: 5px 41px;
|
||
line-height: 16px;
|
||
text-decoration: none;
|
||
"
|
||
>
|
||
<svg
|
||
viewBox="0 0 24 24"
|
||
width="16"
|
||
height="16"
|
||
class="octicon octicon-mark-github"
|
||
>
|
||
<g>
|
||
<path
|
||
d="M4.54,9.46,2.19,7.1a6.93,6.93,0,0,0,0,9.79l2.36-2.36A3.59,3.59,0,0,1,4.54,9.46Z"
|
||
style=""
|
||
fill="#E8710A"
|
||
></path>
|
||
<path
|
||
d="M2.19,7.1,4.54,9.46a3.59,3.59,0,0,1,5.08,0l1.71-2.93h0l-.1-.08h0A6.93,6.93,0,0,0,2.19,7.1Z"
|
||
style=""
|
||
fill="#F9AB00"
|
||
></path>
|
||
<path
|
||
d="M11.34,17.46h0L9.62,14.54a3.59,3.59,0,0,1-5.08,0L2.19,16.9a6.93,6.93,0,0,0,9,.65l.11-.09"
|
||
style=""
|
||
fill="#F9AB00"
|
||
></path>
|
||
<path
|
||
d="M12,7.1a6.93,6.93,0,0,0,0,9.79l2.36-2.36a3.59,3.59,0,1,1,5.08-5.08L21.81,7.1A6.93,6.93,0,0,0,12,7.1Z"
|
||
style=""
|
||
fill="#F9AB00"
|
||
></path>
|
||
<path
|
||
d="M21.81,7.1,19.46,9.46a3.59,3.59,0,0,1-5.08,5.08L12,16.9A6.93,6.93,0,0,0,21.81,7.1Z"
|
||
style=""
|
||
fill="#E8710A"
|
||
></path>
|
||
</g>
|
||
</svg>
|
||
<span> colab</span></a
|
||
>
|
||
</div><form class="sidebar-search-container" method="get" action="../../search.html" role="search">
|
||
<input class="sidebar-search" placeholder=Search name="q" aria-label="Search">
|
||
<input type="hidden" name="check_keywords" value="yes">
|
||
<input type="hidden" name="area" value="default">
|
||
</form>
|
||
<div id="searchbox"></div><div class="sidebar-scroll"><div class="sidebar-tree">
|
||
<p class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
|
||
<ul>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../quickstart/installation.html">Installation</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../quickstart/first_nerf.html">Training your first model</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../quickstart/existing_dataset.html">Using existing data</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../quickstart/custom_dataset.html">Using custom data</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../quickstart/viewer_quickstart.html">Using the viewer</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../quickstart/export_geometry.html">Export geometry</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../quickstart/data_conventions.html">Data conventions</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../reference/contributing.html">Contributing</a></li>
|
||
</ul>
|
||
<p class="caption" role="heading"><span class="caption-text">Extensions</span></p>
|
||
<ul>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../extensions/blender_addon.html">Blender VFX add-on</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../extensions/maya_plugin.html">Autodesk Maya Plug-in</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../extensions/unreal_engine.html">Exporting to Unreal Engine</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../extensions/sdfstudio.html">SDFStudio</a></li>
|
||
</ul>
|
||
<p class="caption" role="heading"><span class="caption-text">NeRFology</span></p>
|
||
<ul class="current">
|
||
<li class="toctree-l1 current has-children"><a class="reference internal" href="index.html">Methods</a><input checked="" class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" role="switch" type="checkbox"/><label for="toctree-checkbox-1"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul class="current">
|
||
<li class="toctree-l2"><a class="reference internal" href="instant_ngp.html"> Instant-NGP</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="splat.html"> Splatfacto</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="splatw.html"> Splatfacto-W</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="in2n.html"> Instruct-NeRF2NeRF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="igs2gs.html"> Instruct-GS2GS</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="signerf.html"> SIGNeRF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="kplanes.html"> K-Planes</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="lerf.html"> LERF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="livescene.html"> LiveScene</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="feature_splatting.html"> Feature-Splatting</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="mipnerf.html"> Mip-NeRF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="nerf.html"> NeRF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="nerfacto.html"> Nerfacto</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="nerfbusters.html"> Nerfbusters</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="nerfplayer.html"> NeRFPlayer</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="tetranerf.html"> Tetra-NeRF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="tensorf.html"> TensoRF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="generfacto.html"> Generfacto</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="pynerf.html"> PyNeRF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="seathru_nerf.html"> SeaThru-NeRF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="zipnerf.html"> Zip-NeRF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="bionerf.html"> BioNeRF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="nerf2gs2nerf.html"> NeRFtoGSandBack</a></li>
|
||
<li class="toctree-l2 current current-page"><a class="current reference internal" href="#"> OpenNeRF</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1 has-children"><a class="reference internal" href="../model_components/index.html">Model components</a><input class="toctree-checkbox" id="toctree-checkbox-2" name="toctree-checkbox-2" role="switch" type="checkbox"/><label for="toctree-checkbox-2"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l2"><a class="reference internal" href="../model_components/visualize_cameras.html"> Cameras models</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../model_components/visualize_samples.html"> Sample representation</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../model_components/visualize_samplers.html"> Ray samplers</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../model_components/visualize_spatial_distortions.html"> Spatial distortions</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../model_components/visualize_encoders.html"> Encoders</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
<p class="caption" role="heading"><span class="caption-text">Developer Guides</span></p>
|
||
<ul>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../developer_guides/new_methods.html">Adding a New Method</a></li>
|
||
<li class="toctree-l1 has-children"><a class="reference internal" href="../../developer_guides/pipelines/index.html">Pipelines overview</a><input class="toctree-checkbox" id="toctree-checkbox-3" name="toctree-checkbox-3" role="switch" type="checkbox"/><label for="toctree-checkbox-3"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l2 has-children"><a class="reference internal" href="../../developer_guides/pipelines/dataparsers.html">DataParsers</a><input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" role="switch" type="checkbox"/><label for="toctree-checkbox-4"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/data/dataparsers.html">Data Parsers</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/pipelines/datamanagers.html">DataManagers</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/pipelines/models.html">Models</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/pipelines/fields.html">Fields</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/pipelines/pipelines.html">Pipelines</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1 has-children"><a class="reference internal" href="../../developer_guides/viewer/index.html">Viewer</a><input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" role="switch" type="checkbox"/><label for="toctree-checkbox-5"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/viewer/custom_gui.html">Custom GUI</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/viewer/viewer_control.html">Python Viewer Control</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/viewer/local_viewer.html">(Legacy Viewer) Local Server</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../developer_guides/config.html">Customizable configs</a></li>
|
||
<li class="toctree-l1 has-children"><a class="reference internal" href="../../developer_guides/debugging_tools/index.html">Debugging tools</a><input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" role="switch" type="checkbox"/><label for="toctree-checkbox-6"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/debugging_tools/local_logger.html">Local writer</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/debugging_tools/profiling.html">Code profiling support</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/debugging_tools/benchmarking.html">Benchmarking workflow</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
<p class="caption" role="heading"><span class="caption-text">Reference</span></p>
|
||
<ul>
|
||
<li class="toctree-l1 has-children"><a class="reference internal" href="../../reference/cli/index.html">CLI</a><input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" role="switch" type="checkbox"/><label for="toctree-checkbox-7"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_install_cli.html">ns-install-cli</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_process_data.html">ns-process-data</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_download_data.html">ns-download-data</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_train.html">ns-train</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_render.html">ns-render</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_viewer.html">ns-viewer</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_export.html">ns-export</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_eval.html">ns-eval</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1 has-children"><a class="reference internal" href="../../reference/api/index.html">API</a><input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" role="switch" type="checkbox"/><label for="toctree-checkbox-8"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/api/cameras.html">Cameras</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/api/config.html">Configs</a></li>
|
||
<li class="toctree-l2 has-children"><a class="reference internal" href="../../reference/api/data/index.html">Data</a><input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" role="switch" type="checkbox"/><label for="toctree-checkbox-9"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/data/dataparsers.html">Data Parsers</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/data/datamanagers.html">Datamanagers</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/data/datasets.html">Datasets</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/data/utils.html">Utils</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/api/fields.html">Fields</a></li>
|
||
<li class="toctree-l2 has-children"><a class="reference internal" href="../../reference/api/field_components/index.html">Field Modules</a><input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" role="switch" type="checkbox"/><label for="toctree-checkbox-10"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/field_components/encodings.html">Encodings</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/field_components/embeddings.html">Embeddings</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/field_components/field_heads.html">Field Heads</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/field_components/mlp.html">MLP</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/field_components/spatial_distortions.html">Spatial Distortions</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/api/models.html">Models</a></li>
|
||
<li class="toctree-l2 has-children"><a class="reference internal" href="../../reference/api/model_components/index.html">Model components</a><input class="toctree-checkbox" id="toctree-checkbox-11" name="toctree-checkbox-11" role="switch" type="checkbox"/><label for="toctree-checkbox-11"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/model_components/ray_sampler.html">Ray Sampler</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/model_components/losses.html">Losses</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/model_components/renderers.html">Renderers</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/api/optimizers.html">Engine</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/api/plugins.html">Plugins</a></li>
|
||
<li class="toctree-l2 has-children"><a class="reference internal" href="../../reference/api/utils/index.html">Utils</a><input class="toctree-checkbox" id="toctree-checkbox-12" name="toctree-checkbox-12" role="switch" type="checkbox"/><label for="toctree-checkbox-12"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/utils/colors.html">Colors</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/utils/math.html">Math</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/utils/colormaps.html">Colormaps</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/utils/tensor_dataclass.html">TensorDataclass</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/api/viewer.html">Viewer</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</aside>
|
||
<div class="main">
|
||
<div class="content">
|
||
<div class="article-container">
|
||
<a href="#" class="back-to-top muted-link">
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24">
|
||
<path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12z"></path>
|
||
</svg>
|
||
<span>Back to top</span>
|
||
</a>
|
||
<div class="content-icon-container">
|
||
|
||
<div class="theme-toggle-container theme-toggle-content">
|
||
<button class="theme-toggle">
|
||
<div class="visually-hidden">Toggle Light / Dark / Auto color theme</div>
|
||
<svg class="theme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
|
||
<svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
</button>
|
||
</div>
|
||
<label class="toc-overlay-icon toc-content-icon" for="__toc">
|
||
<div class="visually-hidden">Toggle table of contents sidebar</div>
|
||
<i class="icon"><svg><use href="#svg-toc"></use></svg></i>
|
||
</label>
|
||
</div>
|
||
<article role="main">
|
||
<section class="tex2jax_ignore mathjax_ignore" id="opennerf">
|
||
<h1>OpenNeRF<a class="headerlink" href="#opennerf" title="Permalink to this heading">#</a></h1>
|
||
<section id="openset-3d-neural-scene-segmentation-with-pixel-wise-features-and-rendered-novel-views">
|
||
<h2>OpenSet 3D Neural Scene Segmentation with Pixel-wise Features and Rendered Novel Views<a class="headerlink" href="#openset-3d-neural-scene-segmentation-with-pixel-wise-features-and-rendered-novel-views" title="Permalink to this heading">#</a></h2>
|
||
<p><a class="sd-sphinx-override sd-btn sd-text-wrap sd-btn-outline-primary reference external" href="https://opennerf.github.io/"><span>Paper Website</span></a></p>
|
||
</section>
|
||
<section id="installation">
|
||
<h2>Installation<a class="headerlink" href="#installation" title="Permalink to this heading">#</a></h2>
|
||
<p>Detailed installation instructions can be found at our <a class="reference external" href="https://github.com/opennerf/opennerf?tab=readme-ov-file#installation">Project Page</a>.</p>
|
||
</section>
|
||
<section id="running-opennerf">
|
||
<h2>Running OpenNeRF<a class="headerlink" href="#running-opennerf" title="Permalink to this heading">#</a></h2>
|
||
<p>Once installed, run:</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>ns-train<span class="w"> </span>opennerf<span class="w"> </span>--help
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="method">
|
||
<h2>Method<a class="headerlink" href="#method" title="Permalink to this heading">#</a></h2>
|
||
<img alt="https://opennerf.github.io/static/images/teaser.png" src="https://opennerf.github.io/static/images/teaser.png" />
|
||
<section id="overview">
|
||
<h3>Overview<a class="headerlink" href="#overview" title="Permalink to this heading">#</a></h3>
|
||
<p>Large visual-language models (VLMs), like CLIP, enable open-set image segmentation to segment arbitrary concepts from an image in a zero-shot manner. This goes beyond the traditional closed-set assumption, i.e., where models can only segment classes from a pre-defined training set. More recently, first works on open-set segmentation in 3D scenes have appeared in the literature. These methods are heavily influenced by closed-set 3D convolutional approaches that process point clouds or polygon meshes. However, these 3D scene representations do not align well with the image-based nature of the visual-language models. Indeed, point cloud and 3D meshes typically have a lower resolution than images and the reconstructed 3D scene geometry might not project well to the underlying 2D image sequences used to compute pixel-aligned CLIP features. To address these challenges, we propose OpenNeRF which naturally operates on posed images and directly encodes the VLM features within the NeRF. This is similar in spirit to LERF, however our work shows that using pixel-wise VLM features (instead of global CLIP features) results in an overall less complex architecture without the need for additional DINO regularization. Our OpenNeRF further leverages NeRF’s ability to render novel views and extract open-set VLM features from areas that are not well observed in the initial posed images. For 3D point cloud segmentation on Replica, OpenNeRF outperforms recent open-vocabulary methods such as LERF and OpenScene by at least +4.9 mIoU.</p>
|
||
</section>
|
||
</section>
|
||
<section id="zero-shot-3d-semantic-segmentation-on-replica">
|
||
<h2>Zero-Shot 3D Semantic Segmentation on Replica<a class="headerlink" href="#zero-shot-3d-semantic-segmentation-on-replica" title="Permalink to this heading">#</a></h2>
|
||
<img alt="https://opennerf.github.io/static/visualizations/final_office0_.gif" src="https://opennerf.github.io/static/visualizations/final_office0_.gif" />
|
||
<img alt="https://opennerf.github.io/static/visualizations/final_office1_.gif" src="https://opennerf.github.io/static/visualizations/final_office1_.gif" />
|
||
<img alt="https://opennerf.github.io/static/visualizations/final_office2_.gif" src="https://opennerf.github.io/static/visualizations/final_office2_.gif" />
|
||
</section>
|
||
<section id="bibtex">
|
||
<h2>BibTeX<a class="headerlink" href="#bibtex" title="Permalink to this heading">#</a></h2>
|
||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nd">@inproceedings</span><span class="p">{</span><span class="n">engelmann2024opennerf</span><span class="p">,</span>
|
||
<span class="n">title</span><span class="o">=</span><span class="p">{{</span><span class="n">OpenNeRF</span><span class="p">:</span> <span class="n">Open</span> <span class="n">Set</span> <span class="mi">3</span><span class="n">D</span> <span class="n">Neural</span> <span class="n">Scene</span> <span class="n">Segmentation</span> <span class="k">with</span> <span class="n">Pixel</span><span class="o">-</span><span class="n">Wise</span> <span class="n">Features</span> <span class="ow">and</span> <span class="n">Rendered</span> <span class="n">Novel</span> <span class="n">Views</span><span class="p">}},</span>
|
||
<span class="n">author</span><span class="o">=</span><span class="p">{</span><span class="n">Engelmann</span><span class="p">,</span> <span class="n">Francis</span> <span class="ow">and</span> <span class="n">Manhardt</span><span class="p">,</span> <span class="n">Fabian</span> <span class="ow">and</span> <span class="n">Niemeyer</span><span class="p">,</span> <span class="n">Michael</span> <span class="ow">and</span> <span class="n">Tateno</span><span class="p">,</span> <span class="n">Keisuke</span> <span class="ow">and</span> <span class="n">Pollefeys</span><span class="p">,</span> <span class="n">Marc</span> <span class="ow">and</span> <span class="n">Tombari</span><span class="p">,</span> <span class="n">Federico</span><span class="p">},</span>
|
||
<span class="n">booktitle</span><span class="o">=</span><span class="p">{</span><span class="n">International</span> <span class="n">Conference</span> <span class="n">on</span> <span class="n">Learning</span> <span class="n">Representations</span><span class="p">},</span>
|
||
<span class="n">year</span><span class="o">=</span><span class="p">{</span><span class="mi">2024</span><span class="p">}</span>
|
||
<span class="p">}</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
|
||
</article>
|
||
</div>
|
||
<footer>
|
||
|
||
<div class="related-pages">
|
||
<a class="next-page" href="../model_components/index.html">
|
||
<div class="page-info">
|
||
<div class="context">
|
||
<span>Next</span>
|
||
</div>
|
||
<div class="title">Model components</div>
|
||
</div>
|
||
<svg class="furo-related-icon"><use href="#svg-arrow-right"></use></svg>
|
||
</a>
|
||
<a class="prev-page" href="nerf2gs2nerf.html">
|
||
<svg class="furo-related-icon"><use href="#svg-arrow-right"></use></svg>
|
||
<div class="page-info">
|
||
<div class="context">
|
||
<span>Previous</span>
|
||
</div>
|
||
|
||
<div class="title">From NeRFs to Gaussian Splats, and Back</div>
|
||
|
||
</div>
|
||
</a>
|
||
</div>
|
||
<div class="bottom-of-page">
|
||
<div class="left-details">
|
||
<div class="copyright">
|
||
Copyright © 2022, nerfstudio Team
|
||
</div>
|
||
Made with <a href="https://www.sphinx-doc.org/">Sphinx</a> and <a class="muted-link" href="https://pradyunsg.me">@pradyunsg</a>'s
|
||
|
||
<a href="https://github.com/pradyunsg/furo">Furo</a>
|
||
|
||
</div>
|
||
<div class="right-details">
|
||
<div class="icons">
|
||
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
</footer>
|
||
</div>
|
||
<aside class="toc-drawer">
|
||
|
||
|
||
<div class="toc-sticky toc-scroll">
|
||
<div class="toc-title-container">
|
||
<span class="toc-title">
|
||
On this page
|
||
</span>
|
||
</div>
|
||
<div class="toc-tree-container">
|
||
<div class="toc-tree">
|
||
<ul>
|
||
<li><a class="reference internal" href="#">OpenNeRF</a><ul>
|
||
<li><a class="reference internal" href="#openset-3d-neural-scene-segmentation-with-pixel-wise-features-and-rendered-novel-views">OpenSet 3D Neural Scene Segmentation with Pixel-wise Features and Rendered Novel Views</a></li>
|
||
<li><a class="reference internal" href="#installation">Installation</a></li>
|
||
<li><a class="reference internal" href="#running-opennerf">Running OpenNeRF</a></li>
|
||
<li><a class="reference internal" href="#method">Method</a><ul>
|
||
<li><a class="reference internal" href="#overview">Overview</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#zero-shot-3d-semantic-segmentation-on-replica">Zero-Shot 3D Semantic Segmentation on Replica</a></li>
|
||
<li><a class="reference internal" href="#bibtex">BibTeX</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
</aside>
|
||
</div>
|
||
</div></body>
|
||
|
||
</html> |