From 7c40594c37ad44e0bb37ab1eebd6e8138a427120 Mon Sep 17 00:00:00 2001 From: Matt Tancik Date: Wed, 8 Feb 2023 19:30:50 -0800 Subject: [PATCH] Mess around with normals --- nerfstudio/configs/method_configs.py | 2 + .../data/dataparsers/nerfstudio_dataparser.py | 2 +- nerfstudio/fields/dreamfusion_field.py | 2 +- nerfstudio/models/dreamfusion.py | 47 ++++++++++--------- nerfstudio/pipelines/dreamfusion_pipeline.py | 4 +- 5 files changed, 32 insertions(+), 25 deletions(-) diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index d468be671..3e450cac5 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -360,6 +360,8 @@ method_configs["dreamfusion"] = DreamfusionTrainerConfig( proposal_warmup=500, proposal_update_every=5, proposal_weights_anneal_max_num_iters=100, + start_lambertian_training=1000, + start_normals_training=500, ), alphas_loss_mult=1.0, opacity_loss_mult=1.0, diff --git a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py index 0980a62ac..b407936a4 100644 --- a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py +++ b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py @@ -46,7 +46,7 @@ class NerfstudioDataParserConfig(DataParserConfig): _target: Type = field(default_factory=lambda: Nerfstudio) """target class to instantiate""" - data: Path = Path("data/nerfstudio/poster") + data: Path = Path("data/nerfstudio") """Directory or explicit json file path specifying location of data.""" scale_factor: float = 1.0 """How much to scale the camera origins by.""" diff --git a/nerfstudio/fields/dreamfusion_field.py b/nerfstudio/fields/dreamfusion_field.py index af2b073c1..f3b61e58f 100644 --- a/nerfstudio/fields/dreamfusion_field.py +++ b/nerfstudio/fields/dreamfusion_field.py @@ -194,7 +194,7 @@ class DreamFusionField(Field): pred_normals_inp = torch.cat([positions_flat, density_embedding.view(-1, self.geo_feat_dim)], dim=-1) x = self.mlp_pred_normals(pred_normals_inp).view(*outputs_shape, -1).to(directions) - outputs[FieldHeadNames.PRED_NORMALS] = math.safe_normalize(self.field_head_pred_normals(x)) + outputs[FieldHeadNames.PRED_NORMALS] = self.field_head_pred_normals(x) h = density_embedding.view(-1, self.geo_feat_dim) diff --git a/nerfstudio/models/dreamfusion.py b/nerfstudio/models/dreamfusion.py index a54bab76d..b8f4a902f 100644 --- a/nerfstudio/models/dreamfusion.py +++ b/nerfstudio/models/dreamfusion.py @@ -61,7 +61,7 @@ class DreamFusionModelConfig(ModelConfig): _target: Type = field(default_factory=lambda: DreamFusionModel) """target class to instantiate""" - orientation_loss_mult: float = 0.01 + orientation_loss_mult: float = 0.0001 """Orientation loss multipier on computed normals.""" pred_normal_loss_mult: float = 0.001 """Predicted normal loss multiplier.""" @@ -321,33 +321,34 @@ class DreamFusionModel(Model): outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i]) normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights) - pred_normals = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights) # .detach() + pred_normals = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights) # lambertian shading - if self.config.random_light_source and self.training: # and if training + if self.config.random_light_source: # and self.training: light_d = ray_bundle.origins[0] + torch.randn(3, dtype=torch.float).to(normals) else: light_d = ray_bundle.origins[0] light_d = math.safe_normalize(light_d) - if self.train_shaded and np.random.random_sample() > 0.75: + if (self.train_shaded and np.random.random_sample() > 0.75) or not self.training: shading_weight = 0.9 else: shading_weight = 0.0 shaded, shaded_albedo = self.shader_lambertian( - rgb=rgb, normals=pred_normals, light_direction=light_d, shading_weight=shading_weight, detach_normals=True + rgb=rgb, normals=pred_normals, light_direction=light_d, shading_weight=shading_weight, detach_normals=False ) outputs["normals"] = self.shader_normals(normals, weights=accum_mask) outputs["pred_normals"] = self.shader_normals(pred_normals, weights=accum_mask) - outputs["shaded"] = accum_mask * shaded + torch.ones_like(shaded) * accum_mask_inv - outputs["shaded_albedo"] = shaded_albedo + outputs["shaded"] = accum_mask * shaded + outputs["other_train_output"] = accum_mask * shaded_albedo + background + outputs["shaded_albedo"] = accum_mask * shaded_albedo outputs["rgb"] = accum_mask * rgb + background - if shading_weight < 0: - if np.random.random_sample() > 0.5: - outputs["train_output"] = shaded + if shading_weight > 0: + if np.random.random_sample() > 0.5 and not self.training: + outputs["train_output"] = outputs["shaded"] else: outputs["train_output"] = accum_mask * shaded_albedo + background else: @@ -355,7 +356,7 @@ class DreamFusionModel(Model): if self.training or True: outputs["rendered_orientation_loss"] = orientation_loss( - weights.detach(), field_outputs[FieldHeadNames.NORMALS], ray_bundle.directions + weights, field_outputs[FieldHeadNames.NORMALS], ray_bundle.directions ) outputs["rendered_pred_normal_loss"] = pred_normal_loss( @@ -374,16 +375,18 @@ class DreamFusionModel(Model): loss_dict = {} loss_dict = misc.scale_dict(loss_dict, self.config.loss_coefficients) - # if self.train_normals: - # orientation loss for computed normals - loss_dict["orientation_loss"] = self.config.orientation_loss_mult * torch.mean( - outputs["rendered_orientation_loss"] - ) - - # ground truth supervision for normals - loss_dict["pred_normal_loss"] = self.config.pred_normal_loss_mult * torch.mean( - outputs["rendered_pred_normal_loss"] - ) + if self.train_normals: + # orientation loss for computed normals + loss_dict["orientation_loss"] = self.config.orientation_loss_mult * torch.mean( + outputs["rendered_orientation_loss"] + ) + # ground truth supervision for normals + loss_dict["pred_normal_loss"] = self.config.pred_normal_loss_mult * torch.mean( + outputs["rendered_pred_normal_loss"] + ) + else: + loss_dict["orientation_loss"] = 0 + loss_dict["pred_normal_loss"] = 0 loss_dict["alphas_loss"] = outputs["alphas_loss"].mean() @@ -423,5 +426,7 @@ class DreamFusionModel(Model): "depth": depth, "prop_depth_0": prop_depth_0, "prop_depth_1": prop_depth_1, + "normals": outputs["normals"], + "pred_normals": outputs["pred_normals"], } return metrics_dict, images_dict diff --git a/nerfstudio/pipelines/dreamfusion_pipeline.py b/nerfstudio/pipelines/dreamfusion_pipeline.py index 02af521ac..42f044a0d 100644 --- a/nerfstudio/pipelines/dreamfusion_pipeline.py +++ b/nerfstudio/pipelines/dreamfusion_pipeline.py @@ -51,7 +51,7 @@ class DreamfusionPipelineConfig(VanillaPipelineConfig): front_prompt: str = ", front view" """appended to prompt for front view""" back_prompt: str = ", back view" - """appended to prompt for back view""" + """appended to prompt for back view""" alphas_penalty: bool = True """enables penalty to encourage sparse weights (penalizing for uniform density along ray)""" alphas_loss_mult: float = 1 @@ -137,7 +137,7 @@ class DreamfusionPipeline(VanillaPipeline): text_embedding = (vert * text_embedding + (90 - vert) * self.top_text_embedding) / 90.0 else: - if batch["vertical"] < 30: + if batch["vertical"] < 40: text_embedding = self.top_text_embedding elif batch["central"] > 315 or batch["central"] <= 45: text_embedding = self.front_text_embedding