mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-13 11:24:00 +08:00
632 lines
40 KiB
HTML
632 lines
40 KiB
HTML
<!doctype html>
|
||
<html class="no-js" lang="en" >
|
||
|
||
<head><meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width,initial-scale=1" />
|
||
<meta name="color-scheme" content="light dark"><meta name="generator" content="Docutils 0.19: https://docutils.sourceforge.io/" />
|
||
|
||
<meta property="og:title" content="Instruct-NeRF2NeRF" />
|
||
|
||
<meta property="og:type" content="website" />
|
||
|
||
<meta property="og:url" content="http://docs.nerf.studio/nerfology/methods/in2n.html" />
|
||
|
||
<meta property="og:description" content="Paper Website Code Instruct-NeRF2NeRF enables instruction-based editing of NeRFs via a 2D diffusion model Installation: First install nerfstudio dependencies. Then run: Running Instruct-NeRF2NeRF: ..." />
|
||
|
||
<meta property="og:image" content="https://assets.nerf.studio/opg.png" />
|
||
|
||
<meta property="og:image:alt" content="Instruct-NeRF2NeRF" />
|
||
<link rel="index" title="Index" href="../../genindex.html" /><link rel="search" title="Search" href="../../search.html" /><link rel="next" title="Instruct-GS2GS" href="igs2gs.html" /><link rel="prev" title="Splatfacto in the Wild" href="splatw.html" />
|
||
|
||
<meta name="generator" content="sphinx-5.2.1, furo 2022.09.29" />
|
||
<title>Instruct-NeRF2NeRF - nerfstudio</title>
|
||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||
<link rel="stylesheet" type="text/css" href="../../_static/styles/furo.css?digest=d81277517bee4d6b0349d71bb2661d4890b5617c" />
|
||
<link rel="stylesheet" type="text/css" href="../../_static/mystnb.css" />
|
||
<link rel="stylesheet" type="text/css" href="../../_static/togglebutton.css" />
|
||
<link rel="stylesheet" type="text/css" href="../../_static/copybutton.css" />
|
||
<link rel="stylesheet" type="text/css" href="../../_static/design-style.b7bb847fb20b106c3d81b95245e65545.min.css" />
|
||
<link rel="stylesheet" type="text/css" href="../../_static/styles/furo-extensions.css?digest=30d1aed668e5c3a91c3e3bf6a60b675221979f0e" />
|
||
|
||
|
||
|
||
|
||
<style>
|
||
body {
|
||
--color-code-background: #f8f9fb;
|
||
--color-code-foreground: black;
|
||
--color-brand-primary: #d34600;
|
||
--color-brand-content: #ff6f00;
|
||
|
||
}
|
||
@media not print {
|
||
body[data-theme="dark"] {
|
||
--color-code-background: #282C34;
|
||
--color-code-foreground: #ABB2BF;
|
||
--color-brand-primary: #fdd06c;
|
||
--color-brand-content: ##fea96a;
|
||
|
||
}
|
||
@media (prefers-color-scheme: dark) {
|
||
body:not([data-theme="light"]) {
|
||
--color-code-background: #282C34;
|
||
--color-code-foreground: #ABB2BF;
|
||
--color-brand-primary: #fdd06c;
|
||
--color-brand-content: ##fea96a;
|
||
|
||
}
|
||
}
|
||
}
|
||
</style><script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||
<script src="../../_static/jquery.js"></script>
|
||
<script src="../../_static/underscore.js"></script>
|
||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||
<script src="../../_static/doctools.js"></script>
|
||
<script src="../../_static/sphinx_highlight.js"></script>
|
||
<script src="../../_static/scripts/furo.js"></script>
|
||
<script src="../../_static/clipboard.min.js"></script>
|
||
<script src="../../_static/copybutton.js"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="../../_static/togglebutton.js"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown, .tag_hide_input div.cell_input, .tag_hide-input div.cell_input, .tag_hide_output div.cell_output, .tag_hide-output div.cell_output, .tag_hide_cell.cell, .tag_hide-cell.cell';</script>
|
||
<script src="../../_static/design-tabs.js"></script>
|
||
<script src="../../_static/require.min.js"></script>
|
||
<script src="../../_static/custom.js"></script>
|
||
</head>
|
||
|
||
<body>
|
||
|
||
<script>
|
||
document.body.dataset.theme = localStorage.getItem("theme") || "auto";
|
||
</script>
|
||
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
|
||
<symbol id="svg-toc" viewBox="0 0 24 24">
|
||
<title>Contents</title>
|
||
<svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 1024 1024">
|
||
<path d="M408 442h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8zm-8 204c0 4.4 3.6 8 8 8h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56zm504-486H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zm0 632H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zM115.4 518.9L271.7 642c5.8 4.6 14.4.5 14.4-6.9V388.9c0-7.4-8.5-11.5-14.4-6.9L115.4 505.1a8.74 8.74 0 0 0 0 13.8z"/>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-menu" viewBox="0 0 24 24">
|
||
<title>Menu</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
|
||
stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-menu">
|
||
<line x1="3" y1="12" x2="21" y2="12"></line>
|
||
<line x1="3" y1="6" x2="21" y2="6"></line>
|
||
<line x1="3" y1="18" x2="21" y2="18"></line>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-arrow-right" viewBox="0 0 24 24">
|
||
<title>Expand</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
|
||
stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-chevron-right">
|
||
<polyline points="9 18 15 12 9 6"></polyline>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun" viewBox="0 0 24 24">
|
||
<title>Light mode</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
|
||
stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" class="feather-sun">
|
||
<circle cx="12" cy="12" r="5"></circle>
|
||
<line x1="12" y1="1" x2="12" y2="3"></line>
|
||
<line x1="12" y1="21" x2="12" y2="23"></line>
|
||
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
|
||
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
|
||
<line x1="1" y1="12" x2="3" y2="12"></line>
|
||
<line x1="21" y1="12" x2="23" y2="12"></line>
|
||
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
|
||
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-moon" viewBox="0 0 24 24">
|
||
<title>Dark mode</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
|
||
stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-moon">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none" />
|
||
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z" />
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun-half" viewBox="0 0 24 24">
|
||
<title>Auto light/dark mode</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
|
||
stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-shadow">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none"/>
|
||
<circle cx="12" cy="12" r="9" />
|
||
<path d="M13 12h5" />
|
||
<path d="M13 15h4" />
|
||
<path d="M13 18h1" />
|
||
<path d="M13 9h4" />
|
||
<path d="M13 6h1" />
|
||
</svg>
|
||
</symbol>
|
||
</svg>
|
||
|
||
<input type="checkbox" class="sidebar-toggle" name="__navigation" id="__navigation">
|
||
<input type="checkbox" class="sidebar-toggle" name="__toc" id="__toc">
|
||
<label class="overlay sidebar-overlay" for="__navigation">
|
||
<div class="visually-hidden">Hide navigation sidebar</div>
|
||
</label>
|
||
<label class="overlay toc-overlay" for="__toc">
|
||
<div class="visually-hidden">Hide table of contents sidebar</div>
|
||
</label>
|
||
|
||
|
||
|
||
<div class="page">
|
||
<header class="mobile-header">
|
||
<div class="header-left">
|
||
<label class="nav-overlay-icon" for="__navigation">
|
||
<div class="visually-hidden">Toggle site navigation sidebar</div>
|
||
<i class="icon"><svg><use href="#svg-menu"></use></svg></i>
|
||
</label>
|
||
</div>
|
||
<div class="header-center">
|
||
<a href="../../index.html"><div class="brand">nerfstudio</div></a>
|
||
</div>
|
||
<div class="header-right">
|
||
<div class="theme-toggle-container theme-toggle-header">
|
||
<button class="theme-toggle">
|
||
<div class="visually-hidden">Toggle Light / Dark / Auto color theme</div>
|
||
<svg class="theme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
|
||
<svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
</button>
|
||
</div>
|
||
<label class="toc-overlay-icon toc-header-icon" for="__toc">
|
||
<div class="visually-hidden">Toggle table of contents sidebar</div>
|
||
<i class="icon"><svg><use href="#svg-toc"></use></svg></i>
|
||
</label>
|
||
</div>
|
||
</header>
|
||
<aside class="sidebar-drawer">
|
||
<div class="sidebar-container">
|
||
|
||
<div class="sidebar-sticky"><a
|
||
class="sidebar-brand"
|
||
href="../../index.html"
|
||
>
|
||
|
||
<div class="sidebar-logo-container">
|
||
<img
|
||
class="sidebar-logo only-light"
|
||
src="../../_static/imgs/logo.png"
|
||
alt="Light Logo"
|
||
/>
|
||
<img
|
||
class="sidebar-logo only-dark"
|
||
src="../../_static/imgs/logo-dark.png"
|
||
alt="Dark Logo"
|
||
/>
|
||
</div>
|
||
</a>
|
||
|
||
<div style="text-align: center">
|
||
<script async defer src="https://buttons.github.io/buttons.js"></script>
|
||
<!-- Place this tag where you want the button to render. -->
|
||
<a
|
||
class="github-button"
|
||
href="https://github.com/nerfstudio-project/nerfstudio/"
|
||
data-color-scheme="no-preference: light; light: light; dark: light;"
|
||
data-size="large"
|
||
data-show-count="true"
|
||
aria-label="Download buttons/github-buttons on GitHub"
|
||
>
|
||
Github
|
||
</a>
|
||
<br />
|
||
<a
|
||
href="https://colab.research.google.com/github/nerfstudio-project/nerfstudio/blob/main/colab/demo.ipynb"
|
||
style="
|
||
background-color: #ebf0f4;
|
||
background-image: linear-gradient(180deg, #f6f8fa, #ebf0f4 90%);
|
||
border-radius: 3px;
|
||
border: 1px solid #1b1f2426;
|
||
display: inline-flex;
|
||
cursor: pointer;
|
||
color: #0d0d0d;
|
||
font-family: Arial;
|
||
font-size: 12px;
|
||
font-weight: bold;
|
||
padding: 5px 41px;
|
||
line-height: 16px;
|
||
text-decoration: none;
|
||
"
|
||
>
|
||
<svg
|
||
viewBox="0 0 24 24"
|
||
width="16"
|
||
height="16"
|
||
class="octicon octicon-mark-github"
|
||
>
|
||
<g>
|
||
<path
|
||
d="M4.54,9.46,2.19,7.1a6.93,6.93,0,0,0,0,9.79l2.36-2.36A3.59,3.59,0,0,1,4.54,9.46Z"
|
||
style=""
|
||
fill="#E8710A"
|
||
></path>
|
||
<path
|
||
d="M2.19,7.1,4.54,9.46a3.59,3.59,0,0,1,5.08,0l1.71-2.93h0l-.1-.08h0A6.93,6.93,0,0,0,2.19,7.1Z"
|
||
style=""
|
||
fill="#F9AB00"
|
||
></path>
|
||
<path
|
||
d="M11.34,17.46h0L9.62,14.54a3.59,3.59,0,0,1-5.08,0L2.19,16.9a6.93,6.93,0,0,0,9,.65l.11-.09"
|
||
style=""
|
||
fill="#F9AB00"
|
||
></path>
|
||
<path
|
||
d="M12,7.1a6.93,6.93,0,0,0,0,9.79l2.36-2.36a3.59,3.59,0,1,1,5.08-5.08L21.81,7.1A6.93,6.93,0,0,0,12,7.1Z"
|
||
style=""
|
||
fill="#F9AB00"
|
||
></path>
|
||
<path
|
||
d="M21.81,7.1,19.46,9.46a3.59,3.59,0,0,1-5.08,5.08L12,16.9A6.93,6.93,0,0,0,21.81,7.1Z"
|
||
style=""
|
||
fill="#E8710A"
|
||
></path>
|
||
</g>
|
||
</svg>
|
||
<span> colab</span></a
|
||
>
|
||
</div><form class="sidebar-search-container" method="get" action="../../search.html" role="search">
|
||
<input class="sidebar-search" placeholder=Search name="q" aria-label="Search">
|
||
<input type="hidden" name="check_keywords" value="yes">
|
||
<input type="hidden" name="area" value="default">
|
||
</form>
|
||
<div id="searchbox"></div><div class="sidebar-scroll"><div class="sidebar-tree">
|
||
<p class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
|
||
<ul>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../quickstart/installation.html">Installation</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../quickstart/first_nerf.html">Training your first model</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../quickstart/existing_dataset.html">Using existing data</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../quickstart/custom_dataset.html">Using custom data</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../quickstart/viewer_quickstart.html">Using the viewer</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../quickstart/export_geometry.html">Export geometry</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../quickstart/data_conventions.html">Data conventions</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../reference/contributing.html">Contributing</a></li>
|
||
</ul>
|
||
<p class="caption" role="heading"><span class="caption-text">Extensions</span></p>
|
||
<ul>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../extensions/blender_addon.html">Blender VFX add-on</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../extensions/maya_plugin.html">Autodesk Maya Plug-in</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../extensions/unreal_engine.html">Exporting to Unreal Engine</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../extensions/sdfstudio.html">SDFStudio</a></li>
|
||
</ul>
|
||
<p class="caption" role="heading"><span class="caption-text">NeRFology</span></p>
|
||
<ul class="current">
|
||
<li class="toctree-l1 current has-children"><a class="reference internal" href="index.html">Methods</a><input checked="" class="toctree-checkbox" id="toctree-checkbox-1" name="toctree-checkbox-1" role="switch" type="checkbox"/><label for="toctree-checkbox-1"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul class="current">
|
||
<li class="toctree-l2"><a class="reference internal" href="instant_ngp.html"> Instant-NGP</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="splat.html"> Splatfacto</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="splatw.html"> Splatfacto-W</a></li>
|
||
<li class="toctree-l2 current current-page"><a class="current reference internal" href="#"> Instruct-NeRF2NeRF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="igs2gs.html"> Instruct-GS2GS</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="signerf.html"> SIGNeRF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="kplanes.html"> K-Planes</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="lerf.html"> LERF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="livescene.html"> LiveScene</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="feature_splatting.html"> Feature-Splatting</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="mipnerf.html"> Mip-NeRF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="nerf.html"> NeRF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="nerfacto.html"> Nerfacto</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="nerfbusters.html"> Nerfbusters</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="nerfplayer.html"> NeRFPlayer</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="tetranerf.html"> Tetra-NeRF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="tensorf.html"> TensoRF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="generfacto.html"> Generfacto</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="pynerf.html"> PyNeRF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="seathru_nerf.html"> SeaThru-NeRF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="zipnerf.html"> Zip-NeRF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="bionerf.html"> BioNeRF</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="nerf2gs2nerf.html"> NeRFtoGSandBack</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="opennerf.html"> OpenNeRF</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1 has-children"><a class="reference internal" href="../model_components/index.html">Model components</a><input class="toctree-checkbox" id="toctree-checkbox-2" name="toctree-checkbox-2" role="switch" type="checkbox"/><label for="toctree-checkbox-2"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l2"><a class="reference internal" href="../model_components/visualize_cameras.html"> Cameras models</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../model_components/visualize_samples.html"> Sample representation</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../model_components/visualize_samplers.html"> Ray samplers</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../model_components/visualize_spatial_distortions.html"> Spatial distortions</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../model_components/visualize_encoders.html"> Encoders</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
<p class="caption" role="heading"><span class="caption-text">Developer Guides</span></p>
|
||
<ul>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../developer_guides/new_methods.html">Adding a New Method</a></li>
|
||
<li class="toctree-l1 has-children"><a class="reference internal" href="../../developer_guides/pipelines/index.html">Pipelines overview</a><input class="toctree-checkbox" id="toctree-checkbox-3" name="toctree-checkbox-3" role="switch" type="checkbox"/><label for="toctree-checkbox-3"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l2 has-children"><a class="reference internal" href="../../developer_guides/pipelines/dataparsers.html">DataParsers</a><input class="toctree-checkbox" id="toctree-checkbox-4" name="toctree-checkbox-4" role="switch" type="checkbox"/><label for="toctree-checkbox-4"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/data/dataparsers.html">Data Parsers</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/pipelines/datamanagers.html">DataManagers</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/pipelines/models.html">Models</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/pipelines/fields.html">Fields</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/pipelines/pipelines.html">Pipelines</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1 has-children"><a class="reference internal" href="../../developer_guides/viewer/index.html">Viewer</a><input class="toctree-checkbox" id="toctree-checkbox-5" name="toctree-checkbox-5" role="switch" type="checkbox"/><label for="toctree-checkbox-5"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/viewer/custom_gui.html">Custom GUI</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/viewer/viewer_control.html">Python Viewer Control</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/viewer/local_viewer.html">(Legacy Viewer) Local Server</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../../developer_guides/config.html">Customizable configs</a></li>
|
||
<li class="toctree-l1 has-children"><a class="reference internal" href="../../developer_guides/debugging_tools/index.html">Debugging tools</a><input class="toctree-checkbox" id="toctree-checkbox-6" name="toctree-checkbox-6" role="switch" type="checkbox"/><label for="toctree-checkbox-6"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/debugging_tools/local_logger.html">Local writer</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/debugging_tools/profiling.html">Code profiling support</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../developer_guides/debugging_tools/benchmarking.html">Benchmarking workflow</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
<p class="caption" role="heading"><span class="caption-text">Reference</span></p>
|
||
<ul>
|
||
<li class="toctree-l1 has-children"><a class="reference internal" href="../../reference/cli/index.html">CLI</a><input class="toctree-checkbox" id="toctree-checkbox-7" name="toctree-checkbox-7" role="switch" type="checkbox"/><label for="toctree-checkbox-7"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_install_cli.html">ns-install-cli</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_process_data.html">ns-process-data</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_download_data.html">ns-download-data</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_train.html">ns-train</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_render.html">ns-render</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_viewer.html">ns-viewer</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_export.html">ns-export</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/cli/ns_eval.html">ns-eval</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1 has-children"><a class="reference internal" href="../../reference/api/index.html">API</a><input class="toctree-checkbox" id="toctree-checkbox-8" name="toctree-checkbox-8" role="switch" type="checkbox"/><label for="toctree-checkbox-8"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/api/cameras.html">Cameras</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/api/config.html">Configs</a></li>
|
||
<li class="toctree-l2 has-children"><a class="reference internal" href="../../reference/api/data/index.html">Data</a><input class="toctree-checkbox" id="toctree-checkbox-9" name="toctree-checkbox-9" role="switch" type="checkbox"/><label for="toctree-checkbox-9"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/data/dataparsers.html">Data Parsers</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/data/datamanagers.html">Datamanagers</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/data/datasets.html">Datasets</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/data/utils.html">Utils</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/api/fields.html">Fields</a></li>
|
||
<li class="toctree-l2 has-children"><a class="reference internal" href="../../reference/api/field_components/index.html">Field Modules</a><input class="toctree-checkbox" id="toctree-checkbox-10" name="toctree-checkbox-10" role="switch" type="checkbox"/><label for="toctree-checkbox-10"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/field_components/encodings.html">Encodings</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/field_components/embeddings.html">Embeddings</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/field_components/field_heads.html">Field Heads</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/field_components/mlp.html">MLP</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/field_components/spatial_distortions.html">Spatial Distortions</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/api/models.html">Models</a></li>
|
||
<li class="toctree-l2 has-children"><a class="reference internal" href="../../reference/api/model_components/index.html">Model components</a><input class="toctree-checkbox" id="toctree-checkbox-11" name="toctree-checkbox-11" role="switch" type="checkbox"/><label for="toctree-checkbox-11"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/model_components/ray_sampler.html">Ray Sampler</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/model_components/losses.html">Losses</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/model_components/renderers.html">Renderers</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/api/optimizers.html">Engine</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/api/plugins.html">Plugins</a></li>
|
||
<li class="toctree-l2 has-children"><a class="reference internal" href="../../reference/api/utils/index.html">Utils</a><input class="toctree-checkbox" id="toctree-checkbox-12" name="toctree-checkbox-12" role="switch" type="checkbox"/><label for="toctree-checkbox-12"><div class="visually-hidden">Toggle child pages in navigation</div><i class="icon"><svg><use href="#svg-arrow-right"></use></svg></i></label><ul>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/utils/colors.html">Colors</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/utils/math.html">Math</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/utils/colormaps.html">Colormaps</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="../../reference/api/utils/tensor_dataclass.html">TensorDataclass</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../../reference/api/viewer.html">Viewer</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</aside>
|
||
<div class="main">
|
||
<div class="content">
|
||
<div class="article-container">
|
||
<a href="#" class="back-to-top muted-link">
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24">
|
||
<path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12z"></path>
|
||
</svg>
|
||
<span>Back to top</span>
|
||
</a>
|
||
<div class="content-icon-container">
|
||
|
||
<div class="theme-toggle-container theme-toggle-content">
|
||
<button class="theme-toggle">
|
||
<div class="visually-hidden">Toggle Light / Dark / Auto color theme</div>
|
||
<svg class="theme-icon-when-auto"><use href="#svg-sun-half"></use></svg>
|
||
<svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
</button>
|
||
</div>
|
||
<label class="toc-overlay-icon toc-content-icon" for="__toc">
|
||
<div class="visually-hidden">Toggle table of contents sidebar</div>
|
||
<i class="icon"><svg><use href="#svg-toc"></use></svg></i>
|
||
</label>
|
||
</div>
|
||
<article role="main">
|
||
<section class="tex2jax_ignore mathjax_ignore" id="instruct-nerf2nerf">
|
||
<h1>Instruct-NeRF2NeRF<a class="headerlink" href="#instruct-nerf2nerf" title="Permalink to this heading">#</a></h1>
|
||
<h4>Editing 3D Scenes with Instructions</h4>
|
||
<p><a class="sd-sphinx-override sd-btn sd-text-wrap sd-btn-outline-primary reference external" href="https://instruct-nerf2nerf.github.io/"><span>Paper Website</span></a></p>
|
||
<p><a class="sd-sphinx-override sd-btn sd-text-wrap sd-btn-outline-primary reference external" href="https://github.com/ayaanzhaque/instruct-nerf2nerf"><span>Code</span></a></p>
|
||
<video id="teaser" muted autoplay playsinline loop controls width="100%">
|
||
<source id="mp4" src="https://instruct-nerf2nerf.github.io/data/videos/face.mp4" type="video/mp4">
|
||
</video>
|
||
<p><strong>Instruct-NeRF2NeRF enables instruction-based editing of NeRFs via a 2D diffusion model</strong></p>
|
||
<section id="installation">
|
||
<h2>Installation<a class="headerlink" href="#installation" title="Permalink to this heading">#</a></h2>
|
||
<p>First install nerfstudio dependencies. Then run:</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>pip<span class="w"> </span>install<span class="w"> </span>git+https://github.com/ayaanzhaque/instruct-nerf2nerf
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="running-instruct-nerf2nerf">
|
||
<h2>Running Instruct-NeRF2NeRF<a class="headerlink" href="#running-instruct-nerf2nerf" title="Permalink to this heading">#</a></h2>
|
||
<p>Details for running Instruct-NeRF2NeRF (built with Nerfstudio!) can be found <a class="reference external" href="https://github.com/ayaanzhaque/instruct-nerf2nerf">here</a>. Once installed, run:</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>ns-train<span class="w"> </span>in2n<span class="w"> </span>--help
|
||
</pre></div>
|
||
</div>
|
||
<p>Three variants of Instruct-NeRF2NeRF are provided:</p>
|
||
<div class="table-wrapper colwidths-auto docutils container">
|
||
<table class="docutils align-default">
|
||
<thead>
|
||
<tr class="row-odd"><th class="head"><p>Method</p></th>
|
||
<th class="head"><p>Description</p></th>
|
||
<th class="head"><p>Memory</p></th>
|
||
<th class="head"><p>Quality</p></th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">in2n</span></code></p></td>
|
||
<td><p>Full model, used in paper</p></td>
|
||
<td><p>~15GB</p></td>
|
||
<td><p>Best</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">in2n-small</span></code></p></td>
|
||
<td><p>Half precision model</p></td>
|
||
<td><p>~12GB</p></td>
|
||
<td><p>Good</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">in2n-tiny</span></code></p></td>
|
||
<td><p>Half prevision with no LPIPS</p></td>
|
||
<td><p>~10GB</p></td>
|
||
<td><p>Ok</p></td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
</section>
|
||
<section id="method">
|
||
<h2>Method<a class="headerlink" href="#method" title="Permalink to this heading">#</a></h2>
|
||
<section id="overview">
|
||
<h3>Overview<a class="headerlink" href="#overview" title="Permalink to this heading">#</a></h3>
|
||
<p>Instruct-NeRF2NeRF is a method for editing NeRF scenes with text-instructions. Given a NeRF of a scene and the collection of images used to reconstruct it, the method uses an image-conditioned diffusion model (<a class="reference external" href="https://www.timothybrooks.com/instruct-pix2pix">InstructPix2Pix</a>) to iteratively edit the input images while optimizing the underlying scene, resulting in an optimized 3D scene that respects the edit instruction. The paper demonstrates that their method is able to edit large-scale, real-world scenes, and is able to accomplish more realistic, targeted edits than prior work.</p>
|
||
</section>
|
||
</section>
|
||
<section id="pipeline">
|
||
<h2>Pipeline<a class="headerlink" href="#pipeline" title="Permalink to this heading">#</a></h2>
|
||
<video id="pipeline" muted autoplay playsinline loop controls width="100%">
|
||
<source id="mp4" src="https://instruct-nerf2nerf.github.io/data/videos/pipeline_animation.mp4" type="video/mp4">
|
||
</video>
|
||
<p>This section will walk through each component of the Instruct-NeRF2NeRF method.</p>
|
||
<section id="how-it-works">
|
||
<h3>How it Works<a class="headerlink" href="#how-it-works" title="Permalink to this heading">#</a></h3>
|
||
<p>Instruct-NeRF2NeRF gradually updates a reconstructed NeRF scene by iteratively updating the dataset images while training the NeRF:</p>
|
||
<ol class="arabic simple">
|
||
<li><p>An image is rendered from the scene at a training viewpoint.</p></li>
|
||
<li><p>It is edited by InstructPix2Pix given a global text instruction.</p></li>
|
||
<li><p>The training dataset image is replaced with the edited image.</p></li>
|
||
<li><p>The NeRF continues training as usual.</p></li>
|
||
</ol>
|
||
</section>
|
||
<section id="editing-images-with-instructpix2pix">
|
||
<h3>Editing Images with InstructPix2Pix<a class="headerlink" href="#editing-images-with-instructpix2pix" title="Permalink to this heading">#</a></h3>
|
||
<p>InstructPix2Pix is an image-editing diffusion model which can be prompted using text instructions. More details on InstructPix2Pix can be found <a class="reference external" href="https://www.timothybrooks.com/instruct-pix2pix">here</a>.</p>
|
||
<p>Traditionally, at inference time, InstructPix2Pix takes as input random noise and is conditioned on an image (the image to edit) and a text instruction. The strength of the edit can be controlled using the image and text classifier-free guidance scales.</p>
|
||
<p>To update a dataset image a given viewpoint, Instruct-NeRF2NeRF first takes the original, unedited training image as image conditioning and uses the global text instruction as text conditioning. The main input to the diffusion model is a noised version of the current render from the given viewpoint. The noise is sampled from a normal distribution and scaled based on a randomly chosen timestep. Then InstructPix2Pix slowly denoises the rendered image by predicting the noised version of the image at previous timesteps until the image is fully denoised. This will produce an edited version of the input image.</p>
|
||
<p>This process mixes the information of the diffusion model, which attempts to edit the image, the current 3D structure of the NeRF, and view-consistent information from the unedited, ground-truth images. By combining this set of information, the edit is respected while maintaining 3D consistency.</p>
|
||
<p>The code snippet for how an image is edited in the pipeline can be found <a class="reference external" href="https://github.com/ayaanzhaque/instruct-nerf2nerf/blob/main/in2n/ip2p.py">here</a>.</p>
|
||
</section>
|
||
<section id="iterative-dataset-update">
|
||
<h3>Iterative Dataset Update<a class="headerlink" href="#iterative-dataset-update" title="Permalink to this heading">#</a></h3>
|
||
<p>When NeRF training starts, the dataset consists of the original, unedited images used to train the original scene. These images are saved separately to use as conditioning for InstructPix2Pix. At each optimization step, some number of NeRF optimization steps are performed, and then some number of images (often just one) are updated. The images are randomly ordered prior to training and then at each step, the images are chosen in order to edit. Once an image has been edited, it is replaced in the dataset. Importantly, at each NeRF step, rays are sampled across the entire dataset, meaning there is a mixed source of supervision between edited images and unedited images. This allows for a gradual optimization that balances maintaining the 3D structure and consistency of the NeRF as well as performing the edit.</p>
|
||
<p>At early iterations of this process, the edited images may be inconsistent with one another, as InstructPix2Pix often doesn’t perform consistent edits across viewpoints. However, over time, since images are edited using the current render of the NeRF, the edits begin to converge towards a globally consistent depiction of the underlying scene. Here is an example of how the underlying dataset evolves and becomes more consistent.</p>
|
||
<video id="idu" muted autoplay playsinline loop controls width="100%">
|
||
<source id="mp4" src="https://instruct-nerf2nerf.github.io/data/videos/du_update.mp4" type="video/mp4">
|
||
</video>
|
||
<p>The traditional method for supervising NeRFs using diffusion models is to use a Score Distillation Sampling (SDS) loss, as proposed in <a class="reference external" href="https://dreamfusion3d.github.io/">DreamFusion</a>. The Iterative Dataset Update method can be viewed as a variant of SDS, as instead of updating a discrete set of images at each step, the loss is a mix of rays from various viewpoints which are edited to varying degrees. The results show that this leads to higher quality performance and more stable optimization.</p>
|
||
</section>
|
||
</section>
|
||
<section id="results">
|
||
<h2>Results<a class="headerlink" href="#results" title="Permalink to this heading">#</a></h2>
|
||
<p>For results, view the <a class="reference external" href="https://instruct-nerf2nerf.github.io/">project page</a>!</p>
|
||
</section>
|
||
</section>
|
||
|
||
</article>
|
||
</div>
|
||
<footer>
|
||
|
||
<div class="related-pages">
|
||
<a class="next-page" href="igs2gs.html">
|
||
<div class="page-info">
|
||
<div class="context">
|
||
<span>Next</span>
|
||
</div>
|
||
<div class="title">Instruct-GS2GS</div>
|
||
</div>
|
||
<svg class="furo-related-icon"><use href="#svg-arrow-right"></use></svg>
|
||
</a>
|
||
<a class="prev-page" href="splatw.html">
|
||
<svg class="furo-related-icon"><use href="#svg-arrow-right"></use></svg>
|
||
<div class="page-info">
|
||
<div class="context">
|
||
<span>Previous</span>
|
||
</div>
|
||
|
||
<div class="title">Splatfacto in the Wild</div>
|
||
|
||
</div>
|
||
</a>
|
||
</div>
|
||
<div class="bottom-of-page">
|
||
<div class="left-details">
|
||
<div class="copyright">
|
||
Copyright © 2022, nerfstudio Team
|
||
</div>
|
||
Made with <a href="https://www.sphinx-doc.org/">Sphinx</a> and <a class="muted-link" href="https://pradyunsg.me">@pradyunsg</a>'s
|
||
|
||
<a href="https://github.com/pradyunsg/furo">Furo</a>
|
||
|
||
</div>
|
||
<div class="right-details">
|
||
<div class="icons">
|
||
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
</footer>
|
||
</div>
|
||
<aside class="toc-drawer">
|
||
|
||
|
||
<div class="toc-sticky toc-scroll">
|
||
<div class="toc-title-container">
|
||
<span class="toc-title">
|
||
On this page
|
||
</span>
|
||
</div>
|
||
<div class="toc-tree-container">
|
||
<div class="toc-tree">
|
||
<ul>
|
||
<li><a class="reference internal" href="#">Instruct-NeRF2NeRF</a><ul>
|
||
<li><a class="reference internal" href="#installation">Installation</a></li>
|
||
<li><a class="reference internal" href="#running-instruct-nerf2nerf">Running Instruct-NeRF2NeRF</a></li>
|
||
<li><a class="reference internal" href="#method">Method</a><ul>
|
||
<li><a class="reference internal" href="#overview">Overview</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#pipeline">Pipeline</a><ul>
|
||
<li><a class="reference internal" href="#how-it-works">How it Works</a></li>
|
||
<li><a class="reference internal" href="#editing-images-with-instructpix2pix">Editing Images with InstructPix2Pix</a></li>
|
||
<li><a class="reference internal" href="#iterative-dataset-update">Iterative Dataset Update</a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#results">Results</a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
</aside>
|
||
</div>
|
||
</div></body>
|
||
|
||
</html> |