nerfstudio/nerfactory/pipelines/base.py

# Copyright 2022 The Plenoptix Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Abstracts for the Pipeline class.
"""

from abc import abstractmethod
from pydoc import locate
from typing import Any, Dict, List, Optional

from torch import nn
from torch.nn import Parameter

from nerfactory.dataloaders.base import Dataloader, setup_dataloader
from nerfactory.models.base import Model, setup_model
from nerfactory.utils import profiler
from nerfactory.utils.config import PipelineConfig


class Pipeline(nn.Module):
    """The intent of this class is to provide a higher level interface for the Model
    that will be easy to use for our Trainer class.

    This class will contain high level functions for the model like getting the loss
    dictionaries and visualization code. It should have ways to get the next iterations
    training loss, evaluation loss, and generate whole images for visualization. Each model
    class should be 1:1 with a pipeline that can act as a standardized interface and hide
    differences in how each model takes in and outputs data.

    This class's function is to hide the dataloader and model classes from the trainer,
    worrying about:
    1) Fetching data with the dataloader
    2) Feeding the model the data and fetching the loss
    Hopefully this provides a higher level interface for the trainer to use, and
    simplifying the model classes, which each may have different forward() methods
    and so on.


    TODO: For visualizer functionality to be added down the line, we should make sure
    that we are still abstracting away the Model from the end user. The visualizer function
    should probably be done by adding a new iterator on the base dataloader that will
    talk to the actual visualizer. This is probably ideal to have the visualizer be
    primarily located in the dataloader (first because it makes sense as the
    visualizers main job in this context is to feed in data for the model to load)
    so that we can have an easier time ensuring that the visualizer is always
    returning the same formatted data as for in train / eval. All this is pending changes to
    be done in the future... but just bear in mind that if learned parameters are in the dataloader,
    the visualizer may have to use those parameters as well.


    Args:
        model: The model to be used in the pipeline.
        dataloader: The dataloader to be used in the pipeline.
        loss_coefficients: A dictionary of loss coefficients that will be used

    Attributes:
        self.dataloader (Dataloader): The dataloader that will be used
        self.model (Model): The model that will be used
    """

    def __init__(self, dataloader: Dataloader, model: Model):
        super().__init__()
        self.dataloader: Dataloader = dataloader
        self.model: Model = model

    @property
    def device(self):
        """Returns the device that the model is on."""
        return self.model.device

    @abstractmethod
    @profiler.time_function
    def get_train_loss_dict(self, step: Optional[int] = None):
        """This function gets your training loss dict. This will be responsible for
        getting the next batch of data from the dataloader and interfacing with the
        Model class, feeding the data to the model's forward function."""
        ray_bundle, batch = self.dataloader.next_train()
        model_outputs, loss_dict, metrics_dict = self.model(ray_bundle, batch)
        return model_outputs, loss_dict, metrics_dict

    @abstractmethod
    @profiler.time_function
    def get_eval_loss_dict(self, step: Optional[int] = None):
        """This function gets your evaluation loss dict. It needs to get the data
        from the dataloader and feed it to the model's forward function"""
        self.eval()
        # NOTE(ethan): next_eval() is not being used right now
        for camera_ray_bundle, batch in self.dataloader.eval_dataloader:
            image_idx = int(camera_ray_bundle.camera_indices[0, 0])
            outputs = self.model.get_outputs_for_camera_ray_bundle(camera_ray_bundle)
            psnr = self.model.log_test_image_outputs(image_idx, step, batch, outputs)
        # TODO(ethan): this function should probably return something?
        self.train()

    @abstractmethod
    def log_test_image_outputs(self) -> None:
        """Log the test image outputs"""

    def load_pipeline(self, loaded_state: Dict[str, Any]) -> None:
        """Load the checkpoint from the given path"""
        state = {key.replace("module.", ""): value for key, value in loaded_state.items()}
        self.load_state_dict(state)  # type: ignore

    def get_param_groups(self) -> Dict[str, List[Parameter]]:
        """Get the param groups for the pipeline.

        Returns:
            A list of dictionaries containing the pipeline's param groups.
        """
        dataloader_params = self.dataloader.get_param_groups()
        model_params = self.model.get_param_groups()
        # TODO(ethan): assert that key names don't overlap
        return {**dataloader_params, **model_params}


@profiler.time_function
def setup_pipeline(config: PipelineConfig, device: str, test_mode=False) -> Pipeline:
    """Setup the pipeline. The dataset inputs should be set with the training data.

    Args:
        config: The pipeline config.
    """
    # dataset_inputs
    dataloader: Dataloader = setup_dataloader(config.dataloader, device=device, test_mode=test_mode)
    # TODO(ethan): get rid of scene_bounds from the model
    model: Model = setup_model(config.model, scene_bounds=dataloader.train_datasetinputs.scene_bounds, device=device)
    pipeline_class = locate(config._target_)  # pylint: disable=protected-access
    pipeline = pipeline_class(dataloader=dataloader, model=model)  # type: ignore
    return pipeline