diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index 7e2ebedeb..d696519af 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -179,6 +179,7 @@ class DataManager(nn.Module): eval_count (int): the step number of our eval iteration, needs to be incremented manually train_dataset (Dataset): the dataset for the train dataset eval_dataset (Dataset): the dataset for the eval dataset + includes_time (bool): whether the dataset includes time information Additional attributes specific to each subclass are defined in the setup_train and setup_eval functions. @@ -189,6 +190,7 @@ class DataManager(nn.Module): eval_dataset: Optional[Dataset] = None train_sampler: Optional[DistributedSampler] = None eval_sampler: Optional[DistributedSampler] = None + includes_time: bool = False def __init__(self): """Constructor for the DataManager class. @@ -407,6 +409,7 @@ class VanillaDataManager(DataManager): # pylint: disable=abstract-method else: self.config.data = self.config.dataparser.data self.dataparser = self.dataparser_config.setup() + self.includes_time = self.dataparser.includes_time self.train_dataparser_outputs = self.dataparser.get_dataparser_outputs(split="train") self.train_dataset = self.create_train_dataset() diff --git a/nerfstudio/data/dataparsers/base_dataparser.py b/nerfstudio/data/dataparsers/base_dataparser.py index 12367e51f..f92b34172 100644 --- a/nerfstudio/data/dataparsers/base_dataparser.py +++ b/nerfstudio/data/dataparsers/base_dataparser.py @@ -127,9 +127,14 @@ class DataParser: Args: config: datasetparser config containing all information needed to instantiate dataset + + Attributes: + config: datasetparser config containing all information needed to instantiate dataset + includes_time: Does the dataset include time information in the camera poses. """ config: DataParserConfig + includes_time: bool = False def __init__(self, config: DataParserConfig): super().__init__() diff --git a/nerfstudio/data/dataparsers/dnerf_dataparser.py b/nerfstudio/data/dataparsers/dnerf_dataparser.py index c26621e92..d7411b4c6 100644 --- a/nerfstudio/data/dataparsers/dnerf_dataparser.py +++ b/nerfstudio/data/dataparsers/dnerf_dataparser.py @@ -53,6 +53,7 @@ class DNeRF(DataParser): """DNeRF Dataset""" config: DNeRFDataParserConfig + includes_time: bool = True def __init__(self, config: DNeRFDataParserConfig): super().__init__(config=config) diff --git a/nerfstudio/data/dataparsers/dycheck_dataparser.py b/nerfstudio/data/dataparsers/dycheck_dataparser.py index 92cca35ff..7ddf051e1 100644 --- a/nerfstudio/data/dataparsers/dycheck_dataparser.py +++ b/nerfstudio/data/dataparsers/dycheck_dataparser.py @@ -202,6 +202,7 @@ class Dycheck(DataParser): """Dycheck (https://arxiv.org/abs/2210.13445) Dataset `iphone` subset""" config: DycheckDataParserConfig + includes_time: bool = True def __init__(self, config: DycheckDataParserConfig): super().__init__(config=config) diff --git a/nerfstudio/viewer/app/package.json b/nerfstudio/viewer/app/package.json index 349b25249..cc5eea7ca 100644 --- a/nerfstudio/viewer/app/package.json +++ b/nerfstudio/viewer/app/package.json @@ -1,7 +1,7 @@ { "name": "viewer", "homepage": ".", - "version": "23-04-10-0", + "version": "23-04-18-0", "private": true, "dependencies": { "@emotion/react": "^11.10.4", diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx index 8a727be27..720501734 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx @@ -48,6 +48,7 @@ import { get_curve_object_from_cameras, get_transform_matrix } from './curve'; import { ViserWebSocketContext, sendWebsocketMessage, + makeThrottledMessageSender, } from '../../WebSocket/ViserWebSocket'; import RenderModal from '../../RenderModal'; import LoadPathModal from '../../LoadPathModal'; @@ -65,6 +66,100 @@ function set_camera_position(camera, matrix) { mat.decompose(camera.position, camera.quaternion, camera.scale); } +function RenderTimeSelector(props) { + const throttled_time_message_sender = props.throttled_time_message_sender; + const disabled = props.disabled; + const isGlobal = props.isGlobal; + const camera = props.camera; + const globalRenderTime = props.globalRenderTime; + const setGlobalRenderTime = props.setGlobalRenderTime; + const applyAll = props.applyAll; + const setAllCameraRenderTime = props.setAllCameraRenderTime; + + const getRenderTimeLabel = () => { + if (!isGlobal) { + return camera.renderTime; + } + camera.renderTime = globalRenderTime; + return globalRenderTime; + }; + + const [UIRenderTime, setUIRenderTime] = React.useState( + isGlobal ? globalRenderTime : getRenderTimeLabel(), + ); + + const [valid, setValid] = React.useState(true); + + useEffect(() => { + setUIRenderTime(getRenderTimeLabel()); + throttled_time_message_sender({ + type: 'TimeConditionMessage', + time: camera.renderTime, + }); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [camera, globalRenderTime]); + + const setRndrTime = (val) => { + if (!isGlobal) { + camera.renderTime = val; + } else { + camera.renderTime = val; + setGlobalRenderTime(val); + } + + if (applyAll) { + setAllCameraRenderTime(val); + } + throttled_time_message_sender({ + type: 'TimeConditionMessage', + time: camera.renderTime, + }); + }; + + const handleValidation = (e) => { + const valueFloat = parseFloat(e.target.value); + let valueStr = String(valueFloat); + if (e.target.value >= 0 && e.target.value <= 1) { + setValid(true); + if (valueFloat === 1.0) { + valueStr = '1.0'; + } + if (valueFloat === 0.0) { + valueStr = '0.0'; + } + setUIRenderTime(parseFloat(valueStr)); + setRndrTime(parseFloat(valueStr)); + } else { + setValid(false); + } + }; + + return ( + setUIRenderTime(e.target.value)} + onBlur={(e) => handleValidation(e)} + disabled={disabled} + sx={{ + input: { + WebkitTextFillColor: `${disabled ? '#24B6FF' : '#EBEBEB'} !important`, + color: `${disabled ? '#24B6FF' : '#EBEBEB'} !important`, + }, + }} + value={UIRenderTime} + error={!valid} + helperText={!valid ? 'RenderTime should be between 0.0 and 1.0' : ''} + variant="standard" + /> + ); +} + function FovSelector(props) { const fovLabel = props.fovLabel; const setFovLabel = props.setFovLabel; @@ -200,6 +295,7 @@ function FovSelector(props) { } function CameraList(props) { + const throttled_time_message_sender = props.throttled_time_message_sender; const sceneTree = props.sceneTree; const cameras = props.cameras; const camera_main = props.camera_main; @@ -211,7 +307,6 @@ function CameraList(props) { const cameraProperties = props.cameraProperties; const setCameraProperties = props.setCameraProperties; const isAnimated = props.isAnimated; - const dispatch = props.dispatch; // eslint-disable-next-line no-unused-vars const slider_value = props.slider_value; const set_slider_value = props.set_slider_value; @@ -411,13 +506,19 @@ function CameraList(props) { fovLabel={fovLabel} setFovLabel={setFovLabel} camera={camera} - dispatch={dispatch} disabled={!isAnimated('FOV')} isGlobal={false} - changeMain={false} /> )} - {!isAnimated('FOV') && ( + {isAnimated('RenderTime') && ( + + )} + {!isAnimated('FOV') && !isAnimated('RenderTime') && (

Animated camera properties will show up here!

@@ -455,7 +556,10 @@ export default function CameraPanel(props) { ); const viser_websocket = useContext(ViserWebSocketContext); + const throttled_time_message_sender = + makeThrottledMessageSender(viser_websocket); const DEFAULT_FOV = 50; + const DEFAULT_RENDER_TIME = 0.0; // react state const [cameras, setCameras] = React.useState([]); @@ -471,6 +575,8 @@ export default function CameraPanel(props) { const [load_path_modal_open, setLoadPathModalOpen] = React.useState(false); const [animate, setAnimate] = React.useState(new Set()); const [globalFov, setGlobalFov] = React.useState(DEFAULT_FOV); + const [globalRenderTime, setGlobalRenderTime] = + React.useState(DEFAULT_RENDER_TIME); // leva store const cameraPropsStore = useCreateStore(); @@ -506,6 +612,10 @@ export default function CameraPanel(props) { const crop_center = useSelector((state) => state.renderingState.crop_center); const crop_scale = useSelector((state) => state.renderingState.crop_scale); + const display_render_time = useSelector( + (state) => state.renderingState.use_time_conditioning, + ); + const setRenderHeight = (value) => { dispatch({ type: 'write', @@ -561,7 +671,7 @@ export default function CameraPanel(props) { const camera_main_copy = camera_main.clone(); camera_main_copy.aspect = 1.0; camera_main_copy.fov = globalFov; - camera_main_copy.renderTime = 0.0; + camera_main_copy.renderTime = globalRenderTime; const new_camera_properties = new Map(); camera_main_copy.properties = new_camera_properties; new_camera_properties.set('FOV', globalFov); @@ -800,14 +910,18 @@ export default function CameraPanel(props) { let lookat = null; let up = null; let fov = null; + let render_time = null; position = curve_object.curve_positions.getPoint(point); lookat = curve_object.curve_lookats.getPoint(point); up = curve_object.curve_ups.getPoint(point); fov = curve_object.curve_fovs.getPoint(point).z; + render_time = curve_object.curve_render_times.getPoint(point).z; + render_time = Math.max(Math.min(render_time, 1.0), 0.0); // clamp time values to [0, 1] const mat = get_transform_matrix(position, lookat, up); set_camera_position(camera_render, mat); setFieldOfView(fov); setGlobalFov(fov); + setGlobalRenderTime(render_time); } // eslint-disable-next-line react-hooks/exhaustive-deps }, [slider_value, render_height, render_width]); @@ -847,11 +961,21 @@ export default function CameraPanel(props) { const mat = get_transform_matrix(position, lookat, up); - camera_path.push({ - camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix - fov, - aspect: camera_render.aspect, - }); + if (display_render_time) { + const renderTime = curve_object.curve_render_times.getPoint(pt).z; + camera_path.push({ + camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix + fov, + aspect: camera_render.aspect, + render_time: Math.max(Math.min(renderTime, 1.0), 0.0), // clamp time values to [0, 1] + }); + } else { + camera_path.push({ + camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix + fov, + aspect: camera_render.aspect, + }); + } } const keyframes = []; @@ -1026,6 +1150,12 @@ export default function CameraPanel(props) { } }; + const setAllCameraRenderTime = (val) => { + for (let i = 0; i < cameras.length; i += 1) { + cameras[i].renderTime = val; + } + }; + return (
@@ -1090,6 +1220,52 @@ export default function CameraPanel(props) { />
+ {display_render_time && ( +
+ + + + +
+ )} {camera_type !== 'equirectangular' && (
@@ -1345,6 +1521,7 @@ export default function CameraPanel(props) {
None: """Callback for when the train speed is changed""" if self.train_speed == "Fast": @@ -135,6 +141,7 @@ class ControlPanel: e.set_hidden(self.output_render == "rgb") for e in self._elements_by_tag["crop"]: e.set_hidden(not self.crop_viewport) + self._time.set_hidden(not self._time_enabled) def update_colormap_options(self, dimensions: int, dtype: type) -> None: """update the colormap options based on the current render @@ -235,3 +242,13 @@ class ControlPanel: def background_color(self, value: Tuple[int, int, int]): """Sets the background color""" self._background_color.value = value + + @property + def time(self) -> float: + """Returns the current background color""" + return self._time.value + + @time.setter + def time(self, value: float): + """Sets the background color""" + self._time.value = value diff --git a/nerfstudio/viewer/server/render_state_machine.py b/nerfstudio/viewer/server/render_state_machine.py index 9a955e42b..e94fc7a2a 100644 --- a/nerfstudio/viewer/server/render_state_machine.py +++ b/nerfstudio/viewer/server/render_state_machine.py @@ -155,7 +155,7 @@ class RenderStateMachine(threading.Thread): cy=intrinsics_matrix[1, 2], camera_type=camera_type, camera_to_worlds=camera_to_world[None, ...], - times=torch.tensor([0.0]), + times=torch.tensor([self.viewer.control_panel.time], dtype=torch.float32), ) camera = camera.to(self.viewer.get_model().device) diff --git a/nerfstudio/viewer/server/viewer_state.py b/nerfstudio/viewer/server/viewer_state.py index b6e96623f..22e0bd021 100644 --- a/nerfstudio/viewer/server/viewer_state.py +++ b/nerfstudio/viewer/server/viewer_state.py @@ -51,6 +51,7 @@ from nerfstudio.viewer.viser.messages import ( IsTrainingMessage, NerfstudioMessage, SaveCheckpointMessage, + TimeConditionMessage, ) if TYPE_CHECKING: @@ -110,6 +111,8 @@ class ViewerState: CONSOLE.print(Panel(table, title="[bold][yellow]Viewer[/bold]", expand=False)) + self.include_time = self.pipeline.datamanager.includes_time + # viewer specific variables self.prev_moving = False self.output_type_changed = True @@ -130,8 +133,13 @@ class ViewerState: self.viser_server.register_handler(CameraPathOptionsRequest, self._handle_camera_path_option_request) self.viser_server.register_handler(CameraPathPayloadMessage, self._handle_camera_path_payload) self.viser_server.register_handler(CropParamsMessage, self._handle_crop_params_message) + if self.include_time: + self.viser_server.use_time_conditioning() + self.viser_server.register_handler(TimeConditionMessage, self._handle_time_condition_message) - self.control_panel = ControlPanel(self._interrupt_render, self._crop_params_update, self._output_type_change) + self.control_panel = ControlPanel( + self.include_time, self._interrupt_render, self._crop_params_update, self._output_type_change + ) self.control_panel.install(self.viser_server) def nested_folder_install(folder_labels: List[str], element: ViewerElement): @@ -233,6 +241,11 @@ class ViewerState: self.control_panel.crop_min = tuple(crop_min.tolist()) self.control_panel.crop_max = tuple(crop_max.tolist()) + def _handle_time_condition_message(self, message: NerfstudioMessage) -> None: + """Handle time conditioning message from viewer.""" + assert isinstance(message, TimeConditionMessage) + self.control_panel.time = message.time + @property def is_training(self) -> bool: """Get is_training flag from viewer.""" diff --git a/nerfstudio/viewer/viser/message_api.py b/nerfstudio/viewer/viser/message_api.py index c0471af69..aa4314627 100644 --- a/nerfstudio/viewer/viser/message_api.py +++ b/nerfstudio/viewer/viser/message_api.py @@ -532,6 +532,10 @@ class MessageApi(abc.ABC): ) return GuiHandle(handle_state) + def use_time_conditioning(self) -> None: + """Use time conditioning.""" + self._queue(messages.UseTimeConditioningMessage()) + def _handle_gui_updates( self: MessageApi, client_id: ClientId, diff --git a/nerfstudio/viewer/viser/messages.py b/nerfstudio/viewer/viser/messages.py index dda627d36..d34efb046 100644 --- a/nerfstudio/viewer/viser/messages.py +++ b/nerfstudio/viewer/viser/messages.py @@ -228,3 +228,16 @@ class StatusMessage(NerfstudioMessage): @dataclasses.dataclass class SaveCheckpointMessage(NerfstudioMessage): """Save checkpoint message.""" + + +@dataclasses.dataclass +class UseTimeConditioningMessage(NerfstudioMessage): + """Use time conditioning message.""" + + +@dataclasses.dataclass +class TimeConditionMessage(NerfstudioMessage): + """Time conditioning message.""" + + time: float + """ Time conditioning value """ diff --git a/pyproject.toml b/pyproject.toml index eb872f3c8..feec29ae7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -144,6 +144,7 @@ disable = [ "too-many-branches", "too-many-instance-attributes", "too-many-locals", + "too-many-statements", "unnecessary-ellipsis", ]