Mess around with normals

This commit is contained in:
Matt Tancik committed 2023-02-08 19:30:50 -08:00
1 parent b3dca69ddb
commit 7c40594c37
5 files changed
+32 -25

No files matched your search

+2
View File
@@ -360,6 +360,8 @@ method_configs["dreamfusion"] = DreamfusionTrainerConfig(
proposal_warmup=500,
proposal_update_every=5,
proposal_weights_anneal_max_num_iters=100,
start_lambertian_training=1000,
start_normals_training=500,
),
alphas_loss_mult=1.0,
opacity_loss_mult=1.0,
@@ -46,7 +46,7 @@ class NerfstudioDataParserConfig(DataParserConfig):
_target: Type = field(default_factory=lambda: Nerfstudio)
"""target class to instantiate"""
data: Path = Path("data/nerfstudio/poster")
data: Path = Path("data/nerfstudio")
"""Directory or explicit json file path specifying location of data."""
scale_factor: float = 1.0
"""How much to scale the camera origins by."""
+1 -1
View File
@@ -194,7 +194,7 @@ class DreamFusionField(Field):
pred_normals_inp = torch.cat([positions_flat, density_embedding.view(-1, self.geo_feat_dim)], dim=-1)
x = self.mlp_pred_normals(pred_normals_inp).view(*outputs_shape, -1).to(directions)
outputs[FieldHeadNames.PRED_NORMALS] = math.safe_normalize(self.field_head_pred_normals(x))
outputs[FieldHeadNames.PRED_NORMALS] = self.field_head_pred_normals(x)
h = density_embedding.view(-1, self.geo_feat_dim)
+26 -21
View File
@@ -61,7 +61,7 @@ class DreamFusionModelConfig(ModelConfig):
_target: Type = field(default_factory=lambda: DreamFusionModel)
"""target class to instantiate"""
orientation_loss_mult: float = 0.01
orientation_loss_mult: float = 0.0001
"""Orientation loss multipier on computed normals."""
pred_normal_loss_mult: float = 0.001
"""Predicted normal loss multiplier."""
@@ -321,33 +321,34 @@ class DreamFusionModel(Model):
outputs[f"prop_depth_{i}"] = self.renderer_depth(weights=weights_list[i], ray_samples=ray_samples_list[i])
normals = self.renderer_normals(normals=field_outputs[FieldHeadNames.NORMALS], weights=weights)
pred_normals = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights) # .detach()
pred_normals = self.renderer_normals(field_outputs[FieldHeadNames.PRED_NORMALS], weights=weights)
# lambertian shading
if self.config.random_light_source and self.training: # and if training
if self.config.random_light_source: # and self.training:
light_d = ray_bundle.origins[0] + torch.randn(3, dtype=torch.float).to(normals)
else:
light_d = ray_bundle.origins[0]
light_d = math.safe_normalize(light_d)
if self.train_shaded and np.random.random_sample() > 0.75:
if (self.train_shaded and np.random.random_sample() > 0.75) or not self.training:
shading_weight = 0.9
else:
shading_weight = 0.0
shaded, shaded_albedo = self.shader_lambertian(
rgb=rgb, normals=pred_normals, light_direction=light_d, shading_weight=shading_weight, detach_normals=True
rgb=rgb, normals=pred_normals, light_direction=light_d, shading_weight=shading_weight, detach_normals=False
)
outputs["normals"] = self.shader_normals(normals, weights=accum_mask)
outputs["pred_normals"] = self.shader_normals(pred_normals, weights=accum_mask)
outputs["shaded"] = accum_mask * shaded + torch.ones_like(shaded) * accum_mask_inv
outputs["shaded_albedo"] = shaded_albedo
outputs["shaded"] = accum_mask * shaded
outputs["other_train_output"] = accum_mask * shaded_albedo + background
outputs["shaded_albedo"] = accum_mask * shaded_albedo
outputs["rgb"] = accum_mask * rgb + background
if shading_weight < 0:
if np.random.random_sample() > 0.5:
outputs["train_output"] = shaded
if shading_weight > 0:
if np.random.random_sample() > 0.5 and not self.training:
outputs["train_output"] = outputs["shaded"]
else:
outputs["train_output"] = accum_mask * shaded_albedo + background
else:
@@ -355,7 +356,7 @@ class DreamFusionModel(Model):
if self.training or True:
outputs["rendered_orientation_loss"] = orientation_loss(
weights.detach(), field_outputs[FieldHeadNames.NORMALS], ray_bundle.directions
weights, field_outputs[FieldHeadNames.NORMALS], ray_bundle.directions
)
outputs["rendered_pred_normal_loss"] = pred_normal_loss(
@@ -374,16 +375,18 @@ class DreamFusionModel(Model):
loss_dict = {}
loss_dict = misc.scale_dict(loss_dict, self.config.loss_coefficients)
# if self.train_normals:
# orientation loss for computed normals
loss_dict["orientation_loss"] = self.config.orientation_loss_mult * torch.mean(
outputs["rendered_orientation_loss"]
)
# ground truth supervision for normals
loss_dict["pred_normal_loss"] = self.config.pred_normal_loss_mult * torch.mean(
outputs["rendered_pred_normal_loss"]
)
if self.train_normals:
# orientation loss for computed normals
loss_dict["orientation_loss"] = self.config.orientation_loss_mult * torch.mean(
outputs["rendered_orientation_loss"]
)
# ground truth supervision for normals
loss_dict["pred_normal_loss"] = self.config.pred_normal_loss_mult * torch.mean(
outputs["rendered_pred_normal_loss"]
)
else:
loss_dict["orientation_loss"] = 0
loss_dict["pred_normal_loss"] = 0
loss_dict["alphas_loss"] = outputs["alphas_loss"].mean()
@@ -423,5 +426,7 @@ class DreamFusionModel(Model):
"depth": depth,
"prop_depth_0": prop_depth_0,
"prop_depth_1": prop_depth_1,
"normals": outputs["normals"],
"pred_normals": outputs["pred_normals"],
}
return metrics_dict, images_dict
+2 -2
View File
@@ -51,7 +51,7 @@ class DreamfusionPipelineConfig(VanillaPipelineConfig):
front_prompt: str = ", front view"
"""appended to prompt for front view"""
back_prompt: str = ", back view"
"""appended to prompt for back view"""
"""appended to prompt for back view"""
alphas_penalty: bool = True
"""enables penalty to encourage sparse weights (penalizing for uniform density along ray)"""
alphas_loss_mult: float = 1
@@ -137,7 +137,7 @@ class DreamfusionPipeline(VanillaPipeline):
text_embedding = (vert * text_embedding + (90 - vert) * self.top_text_embedding) / 90.0
else:
if batch["vertical"] < 30:
if batch["vertical"] < 40:
text_embedding = self.top_text_embedding
elif batch["central"] > 315 or batch["central"] <= 45:
text_embedding = self.front_text_embedding