From 38cbb0f358c46c50859f141ff91677f7bdc8a658 Mon Sep 17 00:00:00 2001 From: Yan Xiao <32349680+AlstonXiao@users.noreply.github.com> Date: Tue, 9 May 2023 09:32:51 -0700 Subject: [PATCH] implementing l1 and tv reg in the original tensorf paper (#1855) * implementing l1 and tv reg in the original tensorf paper * added documentation for tv loss * added test for tv loss * fix formating * fix formatting --------- Co-authored-by: Yan Xiao --- nerfstudio/configs/method_configs.py | 4 +++- nerfstudio/model_components/losses.py | 17 +++++++++++++++ nerfstudio/models/tensorf.py | 31 +++++++++++++++++++++++++-- tests/model_components/test_losses.py | 28 ++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 tests/model_components/test_losses.py diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index f9af96427..2220faceb 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -367,7 +367,9 @@ method_configs["tensorf"] = TrainerConfig( train_num_rays_per_batch=4096, eval_num_rays_per_batch=4096, ), - model=TensoRFModelConfig(), + model=TensoRFModelConfig( + regularization="tv", + ), ), optimizers={ "fields": { diff --git a/nerfstudio/model_components/losses.py b/nerfstudio/model_components/losses.py index 68f11bcc4..43fbbd271 100644 --- a/nerfstudio/model_components/losses.py +++ b/nerfstudio/model_components/losses.py @@ -485,3 +485,20 @@ class ScaleAndShiftInvariantLoss(nn.Module): return self.__prediction_ssi prediction_ssi = property(__get_prediction_ssi) + + +def tv_loss(grids: TensorType["grids", "feature_dim", "row", "column"]) -> TensorType[()]: + """ + https://github.com/apchenstu/TensoRF/blob/4ec894dc1341a2201fe13ae428631b58458f105d/utils.py#L139 + + Args: + grids: stacks of explicit feature grids (stacked at dim 0) + Returns: + average total variation loss for neighbor rows and columns. + """ + number_of_grids = grids.shape[0] + h_tv_count = grids[:, :, 1:, :].shape[1] * grids[:, :, 1:, :].shape[2] * grids[:, :, 1:, :].shape[3] + w_tv_count = grids[:, :, :, 1:].shape[1] * grids[:, :, :, 1:].shape[2] * grids[:, :, :, 1:].shape[3] + h_tv = torch.pow((grids[:, :, 1:, :] - grids[:, :, :-1, :]), 2).sum() + w_tv = torch.pow((grids[:, :, :, 1:] - grids[:, :, :, :-1]), 2).sum() + return 2 * (h_tv / h_tv_count + w_tv / w_tv_count) / number_of_grids diff --git a/nerfstudio/models/tensorf.py b/nerfstudio/models/tensorf.py index f94f2ca58..109c117fb 100644 --- a/nerfstudio/models/tensorf.py +++ b/nerfstudio/models/tensorf.py @@ -43,7 +43,7 @@ from nerfstudio.field_components.encodings import ( ) from nerfstudio.field_components.field_heads import FieldHeadNames from nerfstudio.fields.tensorf_field import TensoRFField -from nerfstudio.model_components.losses import MSELoss +from nerfstudio.model_components.losses import MSELoss, tv_loss from nerfstudio.model_components.ray_samplers import PDFSampler, UniformSampler from nerfstudio.model_components.renderers import ( AccumulationRenderer, @@ -67,7 +67,14 @@ class TensoRFModelConfig(ModelConfig): """final render resolution""" upsampling_iters: Tuple[int, ...] = (2000, 3000, 4000, 5500, 7000) """specifies a list of iteration step numbers to perform upsampling""" - loss_coefficients: Dict[str, float] = to_immutable_dict({"rgb_loss": 1.0}) + loss_coefficients: Dict[str, float] = to_immutable_dict( + { + "rgb_loss": 1.0, + "tv_reg_density": 1e-3, + "tv_reg_color": 1e-4, + "l1_reg": 5e-4, + } + ) """Loss specific weights.""" num_samples: int = 50 """Number of samples in field evaluation""" @@ -80,6 +87,8 @@ class TensoRFModelConfig(ModelConfig): appearance_dim: int = 27 """Number of channels for color encoding""" tensorf_encoding: Literal["triplane", "vm", "cp"] = "vm" + regularization: Literal["none", "l1", "tv"] = "l1" + """Regularization method used in tensorf paper""" class TensoRFModel(Model): @@ -241,6 +250,10 @@ class TensoRFModel(Model): if self.config.enable_collider: self.collider = AABBBoxCollider(scene_box=self.scene_box) + # regularizations + if self.config.tensorf_encoding == "cp" and self.config.regularization == "tv": + raise RuntimeError("TV reg not supported for CP decomposition") + def get_param_groups(self) -> Dict[str, List[Parameter]]: param_groups = {} @@ -295,6 +308,20 @@ class TensoRFModel(Model): rgb_loss = self.rgb_loss(image, outputs["rgb"]) loss_dict = {"rgb_loss": rgb_loss} + + if self.config.regularization == "l1": + l1_parameters = [] + for parameter in self.field.density_encoding.parameters(): + l1_parameters.append(parameter.view(-1)) + loss_dict["l1_reg"] = torch.abs(torch.cat(l1_parameters)).mean() + elif self.config.regularization == "tv": + loss_dict["tv_reg_density"] = tv_loss(self.field.density_encoding.plane_coef) + loss_dict["tv_reg_color"] = tv_loss(self.field.color_encoding.plane_coef) + elif self.config.regularization == "none": + pass + else: + raise ValueError(f"Regularization {self.config.regularization} not supported") + loss_dict = misc.scale_dict(loss_dict, self.config.loss_coefficients) return loss_dict diff --git a/tests/model_components/test_losses.py b/tests/model_components/test_losses.py new file mode 100644 index 000000000..8dbbbc443 --- /dev/null +++ b/tests/model_components/test_losses.py @@ -0,0 +1,28 @@ +""" +Test losses +""" + +import torch + +from nerfstudio.model_components.losses import tv_loss + + +def test_tv_loss(): + """Test total variation loss""" + + grids = torch.ones([3, 64, 128, 128]) + assert tv_loss(grids).item() == 0 + + grids = torch.zeros([1, 10, 100, 100]) + for i in range(100): + for j in range(100): + grid_location = (i + j) % 2 + for k in range(10): + grids[0, k, i, j] = grid_location + + # TV_row = 1, TV_col = 1. Totol tv should be 2 * (TV_row + TV_col) + assert tv_loss(grids).item() == 4.0 + + +if __name__ == "__main__": + test_tv_loss()